Refactor series lookup

crates/nodenet/src/channelconfig.rs

@@ -1,3 +1,4 @@
+use dbconn::worker::PgQueue;
 use err::Error;
 use httpclient::url::Url;
 use netpod::log::*;
@@ -100,13 +101,14 @@ fn channel_config_test_backend(channel: SfDbChannel) -> Result<ChannelTypeConfig
 pub async fn channel_config(
     range: NanoRange,
     channel: SfDbChannel,
+    pgqueue: &PgQueue,
     ncc: &NodeConfigCached,
 ) -> Result<Option<ChannelTypeConfigGen>, Error> {
     if channel.backend() == TEST_BACKEND {
         Ok(Some(channel_config_test_backend(channel)?))
-    } else if ncc.node_config.cluster.scylla.is_some() {
+    } else if ncc.node_config.cluster.scylla_st().is_some() {
         debug!("try to get ChConf for scylla type backend");
-        let ret = scylla_chconf_from_sf_db_channel(range, &channel, ncc)
+        let ret = scylla_chconf_from_sf_db_channel(range, &channel, pgqueue)
             .await
             .map_err(Error::from)?;
         Ok(Some(ChannelTypeConfigGen::Scylla(ret)))
@@ -158,7 +160,7 @@ pub async fn channel_configs(channel: SfDbChannel, ncc: &NodeConfigCached) -> Re
             }
         };
         Ok(ret)
-    } else if ncc.node_config.cluster.scylla.is_some() {
+    } else if ncc.node_config.cluster.scylla_st().is_some() {
         debug!("try to get ChConf for scylla type backend");
         let ret = scylla_all_chconf_from_sf_db_channel(&channel, ncc)
             .await
@@ -206,20 +208,22 @@ pub async fn http_get_channel_config(
 async fn scylla_chconf_from_sf_db_channel(
     range: NanoRange,
     channel: &SfDbChannel,
-    ncc: &NodeConfigCached,
+    pgqueue: &PgQueue,
 ) -> Result<ChConf, Error> {
     if let Some(series) = channel.series() {
-        dbconn::channelconfig::chconf_for_series(channel.backend(), series, ncc).await
+        let ret = pgqueue
+            .chconf_for_series(channel.backend(), series)
+            .await?
+            .recv()
+            .await??;
+        Ok(ret)
     } else {
         // TODO let called function allow to return None instead of error-not-found
-        let ret = dbconn::channelconfig::chconf_best_matching_for_name_and_range(
-            channel.backend(),
-            channel.name(),
-            range,
-            ncc,
-        )
-        .await
-        .map_err(Error::from)?;
+        let ret = pgqueue
+            .chconf_best_matching_name_range_job(channel.backend(), channel.name(), range)
+            .await?
+            .recv()
+            .await??;
         Ok(ret)
     }
 }

crates/nodenet/src/configquorum.rs

@@ -1,4 +1,5 @@
 use crate::channelconfig::http_get_channel_config;
+use dbconn::worker::PgQueue;
 use err::Error;
 use netpod::log::*;
 use netpod::range::evrange::SeriesRange;
@@ -90,13 +91,14 @@ pub async fn find_config_basics_quorum(
     channel: SfDbChannel,
     range: SeriesRange,
     ctx: &ReqCtx,
+    pgqueue: &PgQueue,
     ncc: &NodeConfigCached,
 ) -> Result<Option<ChannelTypeConfigGen>, Error> {
     trace!("find_config_basics_quorum");
     if let Some(_cfg) = &ncc.node.sf_databuffer {
         let channel = if channel.name().is_empty() {
             if let Some(_) = channel.series() {
-                let pgclient = dbconn::create_connection(&ncc.node_config.cluster.database).await?;
+                let (pgclient, _pgjh) = dbconn::create_connection(&ncc.node_config.cluster.database).await?;
                 let pgclient = std::sync::Arc::new(pgclient);
                 dbconn::find_sf_channel_by_series(channel, pgclient)
                     .await
@@ -111,9 +113,9 @@ pub async fn find_config_basics_quorum(
             Some(x) => Ok(Some(ChannelTypeConfigGen::SfDatabuffer(x))),
             None => Ok(None),
         }
-    } else if let Some(_) = &ncc.node_config.cluster.scylla {
+    } else if let Some(_) = &ncc.node_config.cluster.scylla_st() {
         let range = netpod::range::evrange::NanoRange::try_from(&range)?;
-        let ret = crate::channelconfig::channel_config(range, channel, ncc).await?;
+        let ret = crate::channelconfig::channel_config(range, channel, pgqueue, ncc).await?;
         Ok(ret)
     } else {
         Err(Error::with_msg_no_trace(

crates/nodenet/src/conn.rs

@@ -17,6 +17,7 @@ use items_2::empty::empty_events_dyn_ev;
 use items_2::framable::EventQueryJsonStringFrame;
 use items_2::framable::Framable;
 use items_2::frame::decode_frame;
+use items_2::frame::make_error_frame;
 use items_2::frame::make_term_frame;
 use items_2::inmem::InMemoryFrame;
 use netpod::histo::HistoLog2;
@@ -81,7 +82,7 @@ async fn make_channel_events_stream_data(
         let node_count = ncc.node_config.cluster.nodes.len() as u64;
         let node_ix = ncc.ix as u64;
         streams::generators::make_test_channel_events_stream_data(subq, node_count, node_ix)
-    } else if let Some(scyconf) = &ncc.node_config.cluster.scylla {
+    } else if let Some(scyconf) = &ncc.node_config.cluster.scylla_st() {
         let cfg = subq.ch_conf().to_scylla()?;
         scylla_channel_event_stream(subq, cfg, scyconf, ncc).await
     } else if let Some(_) = &ncc.node.channel_archiver {
@@ -125,6 +126,7 @@ pub async fn create_response_bytes_stream(
         return Err(e);
     }
     if evq.is_event_blobs() {
+        // This is only relevant for "api-1" queries in sf-data/imagebuffer based backends.
         // TODO support event blobs as transform
         let fetch_info = evq.ch_conf().to_sf_databuffer()?;
         let stream = disk::raw::conn::make_event_blobs_pipe(&evq, &fetch_info, reqctx, ncc)?;
@@ -151,7 +153,7 @@ pub async fn create_response_bytes_stream(
             })
         });
         // let stream = stream.map(move |x| Box::new(x) as Box<dyn Framable + Send>);
-        let stream = stream.map(|x| x.make_frame().map(|x| x.freeze()));
+        let stream = stream.map(|x| x.make_frame().map(bytes::BytesMut::freeze));
         let ret = Box::pin(stream);
         Ok(ret)
     }

crates/nodenet/src/conn/test.rs

@@ -39,6 +39,7 @@ use tokio::io::AsyncWriteExt;
 use tokio::net::TcpListener;
 use tokio::net::TcpStream;
 
+// TODO unify with Cluster::test_00()
 const TEST_BACKEND: &str = "testbackend-00";
 
 #[test]
@@ -50,22 +51,7 @@ fn raw_data_00() {
         let cfg = NodeConfigCached {
             node_config: NodeConfig {
                 name: "node_name_dummy".into(),
-                cluster: Cluster {
-                    backend: TEST_BACKEND.into(),
-                    nodes: Vec::new(),
-                    database: Database {
-                        name: "".into(),
-                        host: "".into(),
-                        port: 5432,
-                        user: "".into(),
-                        pass: "".into(),
-                    },
-                    run_map_pulse_task: false,
-                    is_central_storage: false,
-                    file_io_buffer_size: FileIoBufferSize(1024 * 8),
-                    scylla: None,
-                    cache_scylla: None,
-                },
+                cluster: Cluster::test_00(),
             },
             node: Node {
                 host: "empty".into(),

crates/nodenet/src/scylla.rs

@@ -29,7 +29,7 @@ pub async fn scylla_channel_event_stream(
     let shape = chconf.shape();
     let do_test_stream_error = false;
     let with_values = evq.need_value_data();
-    debug!("Make EventsStreamScylla for {series:?} {scalar_type:?} {shape:?}");
+    debug!("\n\nmake EventsStreamScylla  {series:?}  {scalar_type:?}  {shape:?}\n");
     let stream = scyllaconn::events::EventsStreamScylla::new(
         series,
         evq.range().into(),