Switch to dyn events process

nodenet/src/channelconfig.rs

@@ -0,0 +1,84 @@
+use err::Error;
+use netpod::log::*;
+use netpod::ChConf;
+use netpod::Channel;
+use netpod::NanoRange;
+use netpod::NodeConfigCached;
+use netpod::ScalarType;
+use netpod::Shape;
+
+pub async fn channel_config(range: NanoRange, channel: Channel, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
+    if channel.backend() == "test-disk-databuffer" {
+        let backend = channel.backend().into();
+        // TODO the series-ids here are just random. Need to integrate with better test setup.
+        let ret = if channel.name() == "scalar-i32-be" {
+            let ret = ChConf {
+                backend,
+                series: 1,
+                name: channel.name().into(),
+                scalar_type: ScalarType::I32,
+                shape: Shape::Scalar,
+            };
+            Ok(ret)
+        } else if channel.name() == "wave-f64-be-n21" {
+            let ret = ChConf {
+                backend,
+                series: 2,
+                name: channel.name().into(),
+                scalar_type: ScalarType::F64,
+                shape: Shape::Wave(21),
+            };
+            Ok(ret)
+        } else if channel.name() == "const-regular-scalar-i32-be" {
+            let ret = ChConf {
+                backend,
+                series: 3,
+                name: channel.name().into(),
+                scalar_type: ScalarType::I32,
+                shape: Shape::Scalar,
+            };
+            Ok(ret)
+        } else {
+            error!("no test information");
+            Err(Error::with_msg_no_trace(format!("no test information"))
+                .add_public_msg("No channel config for test channel {:?}"))
+        };
+        ret
+    } else if channel.backend() == "test-inmem" {
+        let backend = channel.backend().into();
+        let ret = if channel.name() == "inmem-d0-i32" {
+            let ret = ChConf {
+                backend,
+                series: 1,
+                name: channel.name().into(),
+                scalar_type: ScalarType::I32,
+                shape: Shape::Scalar,
+            };
+            Ok(ret)
+        } else {
+            error!("no test information");
+            Err(Error::with_msg_no_trace(format!("no test information"))
+                .add_public_msg("No channel config for test channel {:?}"))
+        };
+        ret
+    } else if ncc.node_config.cluster.scylla.is_some() {
+        info!("try to get ChConf for scylla type backend");
+        let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
+            .await
+            .map_err(Error::from)?;
+        Ok(ret)
+    } else if ncc.node.sf_databuffer.is_some() {
+        info!("try to get ChConf for sf-databuffer type backend");
+        let c1 = disk::channelconfig::config(range, channel, ncc).await?;
+        let ret = ChConf {
+            backend: c1.channel.backend,
+            series: 0,
+            name: c1.channel.name,
+            scalar_type: c1.scalar_type,
+            shape: c1.shape,
+        };
+        Ok(ret)
+    } else {
+        err::todoval()
+    }
+}

nodenet/src/conn.rs

@@ -1,7 +1,6 @@
 use err::Error;
 use futures_util::Stream;
 use futures_util::StreamExt;
-use items::eventfull::EventFull;
 use items::frame::decode_frame;
 use items::frame::make_term_frame;
 use items::EventQueryJsonStringFrame;
@@ -98,7 +97,7 @@ async fn make_channel_events_stream(
         // TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
         let do_one_before_range = false;
         // TODO use better builder pattern with shortcuts for production and dev defaults
-        let f = dbconn::channelconfig::chconf_from_database(evq.channel(), node_config)
+        let f = crate::channelconfig::channel_config(evq.range().clone(), evq.channel().clone(), node_config)
             .await
             .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
         let scyco = conf;
@@ -166,15 +165,6 @@ async fn make_channel_events_stream(
     }
 }
 
-async fn make_event_blobs_stream(
-    evq: PlainEventsQuery,
-    node_config: &NodeConfigCached,
-) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
-    info!("make_event_blobs_stream");
-    let stream = disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await?;
-    Ok(stream)
-}
-
 async fn events_conn_handler_inner_try(
     stream: TcpStream,
     addr: SocketAddr,
@@ -247,7 +237,7 @@ async fn events_conn_handler_inner_try(
 
     let mut stream: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>> =
         if let AggKind::EventBlobs = evq.agg_kind() {
-            match make_event_blobs_stream(evq, node_config).await {
+            match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {
                 Ok(stream) => {
                     let stream = stream.map(|x| Box::new(x) as _);
                     Box::pin(stream)
@@ -273,7 +263,11 @@ async fn events_conn_handler_inner_try(
         let item = item.make_frame();
         match item {
             Ok(buf) => {
-                trace!("write {} bytes", buf.len());
+                if buf.len() > 1024 * 64 {
+                    warn!("emit buf len {}", buf.len());
+                } else {
+                    trace!("emit buf len {}", buf.len());
+                }
                 buf_len_histo.ingest(buf.len() as u32);
                 match netout.write_all(&buf).await {
                     Ok(_) => {}

nodenet/src/nodenet.rs

@@ -1 +1,2 @@
+pub mod channelconfig;
 pub mod conn;