Factor more

disk/src/binned.rs

@@ -391,10 +391,10 @@ pub async fn binned_json(node_config: &NodeConfigCached, query: &BinnedQuery) ->
     Ok(serde_json::to_value(ret)?)
 }
 
-pub trait BinnedStreamKind {
+pub trait BinnedStreamKind: Clone + Unpin + Send + Sync + 'static {
     type BinnedStreamItem: MakeBytesFrame;
     type BinnedStreamType: Stream + Send + 'static;
-    type Dummy: Default + Unpin;
+    type Dummy: Default + Unpin + Send;
 
     fn new_binned_from_prebinned(
         &self,
@@ -413,6 +413,7 @@ pub trait BinnedStreamKind {
     ) -> Result<Self::BinnedStreamType, Error>;
 }
 
+#[derive(Clone)]
 pub struct BinnedStreamKindScalar {}
 
 pub struct BinnedStreamKindWave {}
@@ -449,7 +450,7 @@ impl BinnedStreamKind for BinnedStreamKindScalar {
             query.cache_usage().clone(),
             node_config,
             query.disk_stats_every().clone(),
-            self,
+            self.clone(),
         )?;
         Ok(BinnedStream::new(Box::pin(s))?)
     }

disk/src/binnedstream.rs

@@ -32,7 +32,7 @@ where
         cache_usage: CacheUsage,
         node_config: &NodeConfigCached,
         disk_stats_every: ByteSize,
-        stream_kind: &BK,
+        stream_kind: BK,
     ) -> Result<Self, Error> {
         let patches: Vec<_> = patch_it.collect();
         let mut sp = String::new();
@@ -55,15 +55,15 @@ where
                         cache_usage.clone(),
                         disk_stats_every.clone(),
                     );
-                    let s: Pin<Box<dyn Stream<Item = _> + Send>> =
+                    let ret: Pin<Box<dyn Stream<Item = _> + Send>> =
-                        match PreBinnedScalarValueFetchedStream::new(&query, &node_config) {
+                        match PreBinnedScalarValueFetchedStream::new(&query, &node_config, &stream_kind) {
                             Ok(k) => Box::pin(k),
                             Err(e) => {
                                 error!("error from PreBinnedValueFetchedStream::new {:?}", e);
                                 Box::pin(futures_util::stream::iter(vec![Err(e)]))
                             }
                         };
-                    s
+                    ret
                 }
             })
             .flatten()
@@ -99,9 +99,10 @@ where
                 }
             })
             .into_binned_t(range);
+        let mm = BK::Dummy::default();
         Ok(Self {
             inp: Box::pin(inp),
-            _marker: BK::Dummy::default(),
+            _marker: mm,
         })
     }
 }

disk/src/cache.rs

@@ -1,6 +1,7 @@
 use crate::agg::eventbatch::MinMaxAvgScalarEventBatchStreamItem;
 use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
 use crate::agg::streams::StreamItem;
+use crate::binned::BinnedStreamKind;
 use crate::cache::pbv::PreBinnedValueByteStream;
 use crate::cache::pbvfs::PreBinnedScalarItem;
 use crate::merge::MergedMinMaxAvgScalarStream;
@@ -231,10 +232,14 @@ fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Err
 // NOTE  This answers a request for a single valid pre-binned patch.
 // A user must first make sure that the grid spec is valid, and that this node is responsible for it.
 // Otherwise it is an error.
-pub fn pre_binned_bytes_for_http(
+pub fn pre_binned_bytes_for_http<BK>(
     node_config: &NodeConfigCached,
     query: &PreBinnedQuery,
-) -> Result<PreBinnedValueByteStream, Error> {
+    stream_kind: BK,
+) -> Result<PreBinnedValueByteStream<BK>, Error>
+where
+    BK: BinnedStreamKind,
+{
     if query.channel.backend != node_config.node.backend {
         let err = Error::with_msg(format!(
             "backend mismatch  node: {}  requested: {}",
@@ -249,7 +254,7 @@ pub fn pre_binned_bytes_for_http(
             node_config.ix, patch_node_ix
         )))
     } else {
-        let ret = super::cache::pbv::pre_binned_value_byte_stream_new(query, node_config);
+        let ret = super::cache::pbv::pre_binned_value_byte_stream_new(query, node_config, stream_kind);
         Ok(ret)
     }
 }

disk/src/cache/pbv.rs

@@ -1,6 +1,7 @@
 use crate::agg::binnedt::IntoBinnedT;
 use crate::agg::scalarbinbatch::{MinMaxAvgScalarBinBatch, MinMaxAvgScalarBinBatchStreamItem};
 use crate::agg::streams::StreamItem;
+use crate::binned::BinnedStreamKind;
 use crate::cache::pbvfs::{PreBinnedScalarItem, PreBinnedScalarValueFetchedStream};
 use crate::cache::{CacheFileDesc, MergedFromRemotes, PreBinnedQuery};
 use crate::frame::makeframe::make_frame;
@@ -18,22 +19,32 @@ use std::path::PathBuf;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-pub type PreBinnedValueByteStream = SCC<PreBinnedValueByteStreamInner>;
+pub type PreBinnedValueByteStream<BK> = SCC<PreBinnedValueByteStreamInner<BK>>;
 
-pub struct PreBinnedValueByteStreamInner {
+pub struct PreBinnedValueByteStreamInner<BK>
-    inp: PreBinnedValueStream,
+where
+    BK: BinnedStreamKind,
+{
+    inp: PreBinnedValueStream<BK>,
 }
 
-pub fn pre_binned_value_byte_stream_new(
+pub fn pre_binned_value_byte_stream_new<BK>(
     query: &PreBinnedQuery,
     node_config: &NodeConfigCached,
-) -> PreBinnedValueByteStream {
+    stream_kind: BK,
-    let s1 = PreBinnedValueStream::new(query.clone(), node_config);
+) -> PreBinnedValueByteStream<BK>
+where
+    BK: BinnedStreamKind + Unpin,
+{
+    let s1 = PreBinnedValueStream::new(query.clone(), node_config, stream_kind);
     let s2 = PreBinnedValueByteStreamInner { inp: s1 };
     SCC::new(s2)
 }
 
-impl Stream for PreBinnedValueByteStreamInner {
+impl<BK> Stream for PreBinnedValueByteStreamInner<BK>
+where
+    BK: BinnedStreamKind + Unpin,
+{
     type Item = Result<Bytes, Error>;
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
@@ -49,7 +60,10 @@ impl Stream for PreBinnedValueByteStreamInner {
     }
 }
 
-pub struct PreBinnedValueStream {
+pub struct PreBinnedValueStream<BK>
+where
+    BK: BinnedStreamKind,
+{
     query: PreBinnedQuery,
     node_config: NodeConfigCached,
     open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>,
@@ -65,10 +79,14 @@ pub struct PreBinnedValueStream {
     values: MinMaxAvgScalarBinBatch,
     write_fut: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
     read_cache_fut: Option<Pin<Box<dyn Future<Output = Result<StreamItem<PreBinnedScalarItem>, Error>> + Send>>>,
+    stream_kind: BK,
 }
 
-impl PreBinnedValueStream {
+impl<BK> PreBinnedValueStream<BK>
-    pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached) -> Self {
+where
+    BK: BinnedStreamKind,
+{
+    pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: BK) -> Self {
         Self {
             query,
             node_config: node_config.clone(),
@@ -85,6 +103,7 @@ impl PreBinnedValueStream {
             values: MinMaxAvgScalarBinBatch::empty(),
             write_fut: None,
             read_cache_fut: None,
+            stream_kind,
         }
     }
 
@@ -162,6 +181,7 @@ impl PreBinnedValueStream {
             .map({
                 let q2 = self.query.clone();
                 let disk_stats_every = self.query.disk_stats_every.clone();
+                let stream_kind = self.stream_kind.clone();
                 move |patch| {
                     let query = PreBinnedQuery {
                         patch,
@@ -170,7 +190,7 @@ impl PreBinnedValueStream {
                         cache_usage: q2.cache_usage.clone(),
                         disk_stats_every: disk_stats_every.clone(),
                     };
-                    PreBinnedScalarValueFetchedStream::new(&query, &node_config)
+                    PreBinnedScalarValueFetchedStream::new(&query, &node_config, &stream_kind)
                 }
             })
             .map(|k| {
@@ -199,7 +219,10 @@ impl PreBinnedValueStream {
     }
 }
 
-impl Stream for PreBinnedValueStream {
+impl<BK> Stream for PreBinnedValueStream<BK>
+where
+    BK: BinnedStreamKind,
+{
     // TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
     type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;
 

disk/src/cache/pbvfs.rs

@@ -1,5 +1,6 @@
 use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
 use crate::agg::streams::StreamItem;
+use crate::binned::BinnedStreamKind;
 use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead, PreBinnedQuery};
 use crate::frame::inmem::InMemoryFrameAsyncReadStream;
 use crate::frame::makeframe::decode_frame;
@@ -7,23 +8,29 @@ use err::Error;
 use futures_core::Stream;
 use futures_util::{pin_mut, FutureExt};
 use http::StatusCode;
-#[allow(unused_imports)]
 use netpod::log::*;
 use netpod::{NodeConfigCached, PerfOpts};
 use serde::{Deserialize, Serialize};
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-pub struct PreBinnedScalarValueFetchedStream {
+pub struct PreBinnedScalarValueFetchedStream<BK>
+where
+    BK: BinnedStreamKind,
+{
     uri: http::Uri,
     resfut: Option<hyper::client::ResponseFuture>,
     res: Option<InMemoryFrameAsyncReadStream<HttpBodyAsAsyncRead>>,
     errored: bool,
     completed: bool,
+    stream_kind: BK,
 }
 
-impl PreBinnedScalarValueFetchedStream {
+impl<BK> PreBinnedScalarValueFetchedStream<BK>
-    pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
+where
+    BK: BinnedStreamKind,
+{
+    pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: &BK) -> Result<Self, Error> {
         let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
         let node = &node_config.node_config.cluster.nodes[nodeix as usize];
         let uri: hyper::Uri = format!(
@@ -39,6 +46,7 @@ impl PreBinnedScalarValueFetchedStream {
             res: None,
             errored: false,
             completed: false,
+            stream_kind: stream_kind.clone(),
         };
         Ok(ret)
     }
@@ -50,7 +58,10 @@ pub enum PreBinnedScalarItem {
     RangeComplete,
 }
 
-impl Stream for PreBinnedScalarValueFetchedStream {
+impl<BK> Stream for PreBinnedScalarValueFetchedStream<BK>
+where
+    BK: BinnedStreamKind,
+{
     type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {

httpret/src/lib.rs

@@ -1,4 +1,5 @@
 use bytes::Bytes;
+use disk::binned::BinnedStreamKindScalar;
 use disk::cache::{BinnedQuery, PreBinnedQuery};
 use disk::raw::conn::raw_service;
 use err::Error;
@@ -271,17 +272,22 @@ async fn binned_json(query: BinnedQuery, node_config: &NodeConfigCached) -> Resu
 
 async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
     let (head, _body) = req.into_parts();
-    let q = PreBinnedQuery::from_request(&head)?;
+    let query = PreBinnedQuery::from_request(&head)?;
-    let desc = format!("pre-b-{}", q.patch().bin_t_len() / 1000000000);
+    let desc = format!("pre-b-{}", query.patch().bin_t_len() / 1000000000);
     let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str());
+
+    // TODO factor from the inner scopes the fetch of the channel entry so that I can
+    // provide the stream_kind here:
+    let stream_kind = BinnedStreamKindScalar::new();
+
     span1.in_scope(|| {
-        let ret = match disk::cache::pre_binned_bytes_for_http(node_config, &q) {
+        let ret = match disk::cache::pre_binned_bytes_for_http(node_config, &query, stream_kind) {
             Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(
                 s,
                 format!(
                     "pre-b-{}-p-{}",
-                    q.patch().bin_t_len() / 1000000000,
+                    query.patch().bin_t_len() / 1000000000,
-                    q.patch().patch_beg() / 1000000000,
+                    query.patch().patch_beg() / 1000000000,
                 ),
             ))?,
             Err(e) => {