WIP refactor metrics

netfetch/src/ca/conn.rs

@@ -1112,7 +1112,7 @@ pub struct CaConn {
     read_ioids: HashMap<Ioid, Cid>,
     handler_by_ioid: HashMap<Ioid, Option<Pin<Box<dyn ConnFuture>>>>,
     trace_channel_poll: bool,
-    ca_msg_recv_count: u64,
+    ca_msg_recv_cnt: u64,
     ca_version_recv_count: u64,
     ts_channel_status_pong_last: Instant,
 }
@@ -1179,7 +1179,7 @@ impl CaConn {
             read_ioids: HashMap::new(),
             handler_by_ioid: HashMap::new(),
             trace_channel_poll: false,
-            ca_msg_recv_count: 0,
+            ca_msg_recv_cnt: 0,
             ca_version_recv_count: 0,
             ts_channel_status_pong_last: tsnow,
         }
@@ -2810,9 +2810,9 @@ impl CaConn {
                                 }
                             }
                             CaMsgTy::VersionRes(_) => {
-                                if self.ca_msg_recv_count != 0 {
+                                // TODO must check that version is only ever the first message.
+                                if false {
                                     self.stats.ca_proto_version_later().inc();
-                                    // TODO emit log or count stats
                                 }
                             }
                             CaMsgTy::ChannelCloseRes(x) => {
@@ -2825,12 +2825,11 @@ impl CaConn {
                     }
                     CaItem::Empty => {}
                 }
-                if self.ca_msg_recv_count == 0 {
-                    if self.ca_version_recv_count == 0 {
-                        self.stats.ca_proto_no_version_as_first().inc();
-                    }
+                if false {
+                    // TODO use flags to keep track of whether we have seen version or other msgs.
+                    self.stats.ca_proto_no_version_as_first().inc();
                 }
-                self.ca_msg_recv_count += 1;
+                self.ca_msg_recv_cnt = self.ca_msg_recv_cnt.saturating_add(1);
                 Ready(Some(Ok(())))
             }
             Ready(Some(Err(e))) => {
@@ -3184,17 +3183,14 @@ impl CaConn {
         Ok(())
     }
 
-    fn housekeeping_self(&mut self) {
-        let cnt_max = 0xfffffff000000000;
-        if self.ca_msg_recv_count > cnt_max {
-            let mask = !cnt_max;
-            self.ca_msg_recv_count &= mask;
-        }
-    }
+    fn housekeeping_self(&mut self) {}
 
     fn metrics_emit(&mut self) {
+        let ca_msg_recv_cnt = self.ca_msg_recv_cnt;
+        self.ca_msg_recv_cnt = 0;
         let item = CaConnMetrics {
             ca_conn_event_out_queue_len: self.ca_conn_event_out_queue.len(),
+            ca_msg_recv_cnt,
         };
         let item = CaConnEvent::new(Instant::now(), CaConnEventValue::Metrics(item));
         self.ca_conn_event_out_queue.push_back(item);

netfetch/src/ca/connset.rs

@@ -59,6 +59,8 @@ use std::net::SocketAddr;
 use std::net::SocketAddrV4;
 use std::pin::Pin;
 
+use crate::metrics::types::CaConnMetricsAgg;
+use crate::metrics::types::CaConnSetMetrics;
 use crate::queueset::QueueSet;
 use netpod::OnDrop;
 use netpod::TsNano;
@@ -257,6 +259,7 @@ impl CaConnSetEvent {
 pub enum CaConnSetItem {
     Error(Error),
     Healthy,
+    Metrics(crate::metrics::types::CaConnSetMetrics),
 }
 
 pub struct CaConnSetCtrl {
@@ -457,6 +460,7 @@ pub struct CaConnSet {
     rogue_channel_count: u64,
     connect_fail_count: usize,
     cssid_latency_max: Duration,
+    ca_connset_metrics: CaConnSetMetrics,
 }
 
 impl CaConnSet {
@@ -530,6 +534,7 @@ impl CaConnSet {
             rogue_channel_count: 0,
             connect_fail_count: 0,
             cssid_latency_max: Duration::from_millis(2000),
+            ca_connset_metrics: CaConnSetMetrics::new(),
         };
         // TODO await on jh
         let jh = tokio::spawn(CaConnSet::run(connset));
@@ -765,7 +770,7 @@ impl CaConnSet {
             CaConnEventValue::EndOfStream(reason) => self.handle_ca_conn_eos(addr, reason),
             CaConnEventValue::ChannelRemoved(name) => self.handle_ca_conn_channel_removed(addr, name),
             CaConnEventValue::Metrics(v) => {
-                // TODO aggregate metrics and stats
+                self.ca_connset_metrics.ca_conn_agg.ingest(v);
                 Ok(())
             }
         }
@@ -1440,7 +1445,8 @@ impl CaConnSet {
                 | CaConnEventValue::EchoTimeout
                 | CaConnEventValue::ConnCommandResult(..)
                 | CaConnEventValue::ChannelCreateFail(..)
-                | CaConnEventValue::ChannelStatus(..) => {
+                | CaConnEventValue::ChannelStatus(..)
+                | CaConnEventValue::Metrics(..) => {
                     if let Err(e) = tx1.send((addr, item)).await {
                         error!("channel send  {:?}", e);
                         return Err(e.into());
@@ -1456,9 +1462,6 @@ impl CaConnSet {
                         return Err(e.into());
                     }
                 }
-                CaConnEventValue::Metrics(_) => {
-                    // TODO merge metrics
-                }
             }
         }
         if let Some(x) = eos_reason {
@@ -1878,6 +1881,11 @@ impl CaConnSet {
                 self.storage_insert_lt_qu.push_back(a);
             }
         }
+        {
+            let metrics = std::mem::replace(&mut self.ca_connset_metrics, CaConnSetMetrics::new());
+            let item = CaConnSetItem::Metrics(metrics);
+            self.connset_out_queue.push_back(item);
+        }
         Ok(())
     }
 }
@@ -2009,10 +2017,6 @@ impl Stream for CaConnSet {
                 break Ready(Some(CaConnSetItem::Error(e)));
             }
 
-            if let Some(item) = self.connset_out_queue.pop_front() {
-                break Ready(Some(item));
-            }
-
             match self.ticker.poll_unpin(cx) {
                 Ready(()) => match self.as_mut().handle_own_ticker_tick(cx) {
                     Ok(()) => {
@@ -2028,6 +2032,10 @@ impl Stream for CaConnSet {
                 }
             }
 
+            if let Some(item) = self.connset_out_queue.pop_front() {
+                break Ready(Some(item));
+            }
+
             if let Some((addr, jh)) = self.await_ca_conn_jhs.front_mut() {
                 match jh.poll_unpin(cx) {
                     Ready(x) => {