Glue to transition to different events container

disk/src/raw/conn.rs

@@ -7,6 +7,8 @@ use futures_util::{Stream, StreamExt};
 use items::eventfull::EventFull;
 use items::numops::{BoolNum, NumOps, StringNum};
 use items::{EventsNodeProcessor, Framable, RangeCompletableItem, Sitemty, StreamItem};
+use items_2::channelevents::ChannelEvents;
+use items_2::eventsdim0::EventsDim0;
 use netpod::log::*;
 use netpod::query::RawEventsQuery;
 use netpod::{AggKind, ByteOrder, ByteSize, Channel, DiskIoTune, NanoRange, NodeConfigCached, ScalarType, Shape};
@@ -19,7 +21,7 @@ fn make_num_pipeline_stream_evs<NTY, END, EVS, ENP>(
     event_value_shape: EVS,
     events_node_proc: ENP,
     event_blobs: EventChunkerMultifile,
-) -> Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>
+) -> Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>
 where
     NTY: NumOps + NumFromBytes<NTY, END> + 'static,
     END: Endianness + 'static,
@@ -33,17 +35,35 @@ where
         Ok(item) => match item {
             StreamItem::DataItem(item) => match item {
                 RangeCompletableItem::Data(item) => {
-                    let item = events_node_proc.process(item);
+                    // TODO fix super ugly slow glue code
                     use items::EventsNodeProcessorOutput;
-                    let parts = item.into_parts::<NTY>();
+                    let mut item = events_node_proc.process(item);
-                    let item = items_2::eventsdim0::EventsDim0 {
+                    if let Some(item) = item
-                        tss: parts.1,
+                        .as_any_mut()
-                        pulses: VecDeque::new(),
+                        .downcast_mut::<items::scalarevents::ScalarEvents<NTY>>()
-                        values: parts.0,
+                    {
-                    };
+                        warn!("ScalarEvents");
-                    let item = Box::new(item) as Box<dyn items_0::Events>;
+                        let tss: VecDeque<u64> = item.tss.iter().map(|x| *x).collect();
-                    //Ok(StreamItem::DataItem(RangeCompletableItem::Data(todo!())))
+                        let pulses: VecDeque<u64> = item.pulses.iter().map(|x| *x).collect();
-                    Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
+                        let values: VecDeque<NTY> = item.values.iter().map(|x| x.clone()).collect();
+                        let item = EventsDim0 { tss, pulses, values };
+                        let item = ChannelEvents::Events(Box::new(item));
+                        Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
+                    } else {
+                        if let Some(item) = item.as_any_mut().downcast_mut::<items::waveevents::WaveEvents<NTY>>() {
+                            warn!("WaveEvents");
+                            let _tss: VecDeque<u64> = item.tss.iter().map(|x| *x).collect();
+                            let _pulses: VecDeque<u64> = item.pulses.iter().map(|x| *x).collect();
+                            let _values: VecDeque<Vec<NTY>> = item.vals.iter().map(|x| x.clone()).collect();
+                            //let item = EventsDim1 { tss, pulses, values };
+                            //let item = ChannelEvents::Events(Box::new(item));
+                            //Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
+                            Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
+                        } else {
+                            error!("TODO bad, no idea what this item is");
+                            Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
+                        }
+                    }
                 }
                 RangeCompletableItem::RangeComplete => Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)),
             },
@@ -51,8 +71,7 @@ where
             StreamItem::Stats(item) => Ok(StreamItem::Stats(item)),
         },
         Err(e) => Err(e),
-    })
+    });
-    .map(|item| Box::new(item) as Box<dyn Framable + Send>);
     Box::pin(s2)
 }
 
@@ -150,7 +169,7 @@ macro_rules! pipe1 {
 pub async fn make_event_pipe(
     evq: &RawEventsQuery,
     node_config: &NodeConfigCached,
-) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
+) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
     if false {
         match dbconn::channel_exists(&evq.channel, &node_config).await {
             Ok(_) => (),

items/src/items.rs

@@ -368,7 +368,8 @@ impl FrameType for EventQueryJsonStringFrame {
 pub trait EventsNodeProcessorOutput:
     Send + Unpin + DeserializeOwned + WithTimestamps + TimeBinnableType + ByteEstimate
 {
-    fn into_parts<NTY>(self) -> (VecDeque<NTY>, VecDeque<u64>);
+    fn as_any_mut(&mut self) -> &mut dyn Any;
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>);
 }
 
 pub trait EventsNodeProcessor: Send + Unpin {

items/src/scalarevents.rs

@@ -827,7 +827,15 @@ impl<NTY> EventsNodeProcessorOutput for ScalarEvents<NTY>
 where
     NTY: NumOps,
 {
-    fn into_parts<NTY2>(self) -> (VecDeque<NTY2>, VecDeque<u64>) {
+    fn as_any_mut(&mut self) -> &mut dyn Any {
-        todo!()
+        self
+    }
+
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>) {
+        (
+            Box::new(VecDeque::from(self.values)),
+            self.tss.into(),
+            self.pulses.into(),
+        )
     }
 }

items/src/statsevents.rs

@@ -8,6 +8,7 @@ use err::Error;
 use netpod::log::*;
 use netpod::{NanoRange, Shape};
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::collections::VecDeque;
 use std::fmt;
 use tokio::fs::File;
@@ -423,7 +424,11 @@ impl EventAppendable for StatsEvents {
 }
 
 impl EventsNodeProcessorOutput for StatsEvents {
-    fn into_parts<NTY2>(self) -> (VecDeque<NTY2>, VecDeque<u64>) {
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>) {
         todo!()
     }
 }

items/src/waveevents.rs

@@ -12,6 +12,7 @@ use items_0::subfr::SubFrId;
 use netpod::log::*;
 use netpod::{x_bin_count, AggKind, NanoRange, Shape};
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::collections::VecDeque;
 use std::marker::PhantomData;
 use tokio::fs::File;
@@ -540,7 +541,11 @@ impl<NTY> EventsNodeProcessorOutput for WaveEvents<NTY>
 where
     NTY: NumOps,
 {
-    fn into_parts<NTY2>(self) -> (VecDeque<NTY2>, VecDeque<u64>) {
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>) {
         todo!()
     }
 }

items/src/xbinnedscalarevents.rs

@@ -1,3 +1,4 @@
+use std::any::Any;
 use std::collections::VecDeque;
 
 use crate::binsdim0::MinMaxAvgDim0Bins;
@@ -509,7 +510,11 @@ impl<NTY> EventsNodeProcessorOutput for XBinnedScalarEvents<NTY>
 where
     NTY: NumOps,
 {
-    fn into_parts<NTY2>(self) -> (VecDeque<NTY2>, VecDeque<u64>) {
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>) {
         todo!()
     }
 }

items/src/xbinnedwaveevents.rs

@@ -12,6 +12,7 @@ use netpod::log::*;
 use netpod::timeunits::*;
 use netpod::{NanoRange, Shape};
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::collections::VecDeque;
 use std::mem;
 use tokio::fs::File;
@@ -539,7 +540,11 @@ impl<NTY> EventsNodeProcessorOutput for XBinnedWaveEvents<NTY>
 where
     NTY: NumOps,
 {
-    fn into_parts<NTY2>(self) -> (VecDeque<NTY2>, VecDeque<u64>) {
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn into_parts(self) -> (Box<dyn Any>, VecDeque<u64>, VecDeque<u64>) {
         todo!()
     }
 }

nodenet/Cargo.toml

@@ -19,7 +19,6 @@ bytes = "1.0.1"
 crc32fast = "1.2.1"
 arrayref = "0.3.6"
 byteorder = "1.4.3"
-futures-core = "0.3.14"
 futures-util = "0.3.14"
 tracing = "0.1.25"
 hex = "0.4.3"

nodenet/src/conn.rs

@@ -1,8 +1,8 @@
 use err::Error;
-use futures_core::Stream;
+use futures_util::{stream, Stream, StreamExt};
-use futures_util::StreamExt;
 use items::frame::{decode_frame, make_term_frame};
 use items::{EventQueryJsonStringFrame, Framable, RangeCompletableItem, Sitemty, StreamItem};
+use items_0::Empty;
 use items_2::channelevents::ChannelEvents;
 use netpod::histo::HistoLog2;
 use netpod::log::*;
@@ -107,107 +107,114 @@ async fn events_conn_handler_inner_try(
         return Err((e, netout).into());
     }
 
-    let mut p1: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>> =
+    let mut p1: Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>> = if evq.channel().backend() == "test-inmem"
-        if evq.channel().backend() == "test-inmem" {
+    {
-            warn!("TEST BACKEND DATA");
+        warn!("TEST BACKEND DATA");
-            use items_0::Empty;
+        use netpod::timeunits::MS;
-            use netpod::timeunits::MS;
+        let node_count = node_config.node_config.cluster.nodes.len();
-            let node_count = node_config.node_config.cluster.nodes.len();
+        let node_ix = node_config.ix;
-            let node_ix = node_config.ix;
+        if evq.channel().name() == "inmem-d0-i32" {
-            if evq.channel().name() == "inmem-d0-i32" {
+            let mut item = items_2::eventsdim0::EventsDim0::<i32>::empty();
-                let mut item = items_2::eventsdim0::EventsDim0::<i32>::empty();
+            let td = MS * 10;
-                let td = MS * 10;
+            for i in 0..20 {
-                for i in 0..20 {
+                let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
-                    let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
+                let pulse = 1 + node_ix as u64 + node_count as u64 * i;
-                    let pulse = 1 + node_ix as u64 + node_count as u64 * i;
+                item.push(ts, pulse, pulse as _);
-                    item.push(ts, pulse, pulse as _);
-                }
-                let item = ChannelEvents::Events(Box::new(item) as _);
-                let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
-                let item = Box::new(item) as _;
-                let stream = futures_util::stream::iter([item]);
-                Box::pin(stream)
-            } else if evq.channel().name() == "inmem-d0-f32" {
-                let mut item = items_2::eventsdim0::EventsDim0::<f32>::empty();
-                let td = MS * 10;
-                for i in 0..20 {
-                    let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
-                    let pulse = 1 + node_ix as u64 + node_count as u64 * i;
-                    item.push(ts, pulse, ts as _);
-                }
-                let item = ChannelEvents::Events(Box::new(item) as _);
-                let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
-                let item = Box::new(item) as _;
-                let stream = futures_util::stream::iter([item]);
-                Box::pin(stream)
-            } else {
-                let stream = futures_util::stream::empty();
-                Box::pin(stream)
             }
-        } else if let Some(conf) = &node_config.node_config.cluster.scylla {
+            let item = ChannelEvents::Events(Box::new(item) as _);
-            // TODO depends in general on the query
+            let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
-            // TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
+            let stream = futures_util::stream::iter([item]);
-            let do_one_before_range = false;
+            Box::pin(stream)
-            // TODO use better builder pattern with shortcuts for production and dev defaults
+        } else if evq.channel().name() == "inmem-d0-f32" {
-            let qu = PlainEventsQuery::new(evq.channel, evq.range, 1024 * 8, None, true);
+            let mut item = items_2::eventsdim0::EventsDim0::<f32>::empty();
-            let scyco = conf;
+            let td = MS * 10;
-            let _dbconf = node_config.node_config.cluster.database.clone();
+            for i in 0..20 {
-            let scy = match scyllaconn::create_scy_session(scyco).await {
+                let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
-                Ok(k) => k,
+                let pulse = 1 + node_ix as u64 + node_count as u64 * i;
-                Err(e) => return Err((e, netout))?,
+                item.push(ts, pulse, ts as _);
-            };
+            }
-            let series = err::todoval();
+            let item = ChannelEvents::Events(Box::new(item) as _);
-            let scalar_type = err::todoval();
+            let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
-            let shape = err::todoval();
+            let stream = futures_util::stream::iter([item]);
-            let do_test_stream_error = false;
+            Box::pin(stream)
-            let stream = match scyllaconn::events::make_scylla_stream(
-                &qu,
-                do_one_before_range,
-                series,
-                scalar_type,
-                shape,
-                scy,
-                do_test_stream_error,
-            )
-            .await
-            {
-                Ok(k) => k,
-                Err(e) => return Err((e, netout))?,
-            };
-            let s = stream.map(|item| {
-                let item = match item {
-                    Ok(item) => match item {
-                        ChannelEvents::Events(_item) => {
-                            // TODO
-                            let item = items::scalarevents::ScalarEvents::<f64>::empty();
-                            Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
-                        }
-                        ChannelEvents::Status(_item) => todo!(),
-                    },
-                    Err(e) => Err(e),
-                };
-                Box::new(item) as Box<dyn Framable + Send>
-            });
-            Box::pin(s)
-        } else if let Some(_) = &node_config.node.channel_archiver {
-            let e = Error::with_msg_no_trace("archapp not built");
-            return Err((e, netout))?;
-        } else if let Some(_) = &node_config.node.archiver_appliance {
-            let e = Error::with_msg_no_trace("archapp not built");
-            return Err((e, netout))?;
         } else {
-            let stream = match evq.agg_kind {
+            let stream = futures_util::stream::empty();
-                AggKind::EventBlobs => match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {
+            Box::pin(stream)
-                    Ok(j) => j,
+        }
-                    Err(e) => return Err((e, netout))?,
+    } else if let Some(conf) = &node_config.node_config.cluster.scylla {
-                },
+        // TODO depends in general on the query
-                _ => match disk::raw::conn::make_event_pipe(&evq, node_config).await {
+        // TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
-                    Ok(j) => j,
+        let do_one_before_range = false;
-                    Err(e) => return Err((e, netout))?,
+        // TODO use better builder pattern with shortcuts for production and dev defaults
-                },
+        let qu = PlainEventsQuery::new(evq.channel, evq.range, 1024 * 8, None, true);
-            };
+        let scyco = conf;
-            stream
+        let _dbconf = node_config.node_config.cluster.database.clone();
+        let scy = match scyllaconn::create_scy_session(scyco).await {
+            Ok(k) => k,
+            Err(e) => return Err((e, netout))?,
         };
+        let series = err::todoval();
+        let scalar_type = err::todoval();
+        let shape = err::todoval();
+        let do_test_stream_error = false;
+        let stream = match scyllaconn::events::make_scylla_stream(
+            &qu,
+            do_one_before_range,
+            series,
+            scalar_type,
+            shape,
+            scy,
+            do_test_stream_error,
+        )
+        .await
+        {
+            Ok(k) => k,
+            Err(e) => return Err((e, netout))?,
+        };
+        let e = Error::with_msg_no_trace("TODO scylla events");
+        if true {
+            return Err((e, netout))?;
+        }
+        let _s = stream.map(|item| {
+            let item = match item {
+                Ok(item) => match item {
+                    ChannelEvents::Events(_item) => {
+                        // TODO
+                        let item = items_2::eventsdim0::EventsDim0::<f64>::empty();
+                        let item = ChannelEvents::Events(Box::new(item));
+                        let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
+                        item
+                    }
+                    ChannelEvents::Status(_item) => todo!(),
+                },
+                Err(e) => Err(e),
+            };
+            Box::new(item) as Box<dyn Framable + Send>
+        });
+        let s = stream::empty();
+        Box::pin(s)
+    } else if let Some(_) = &node_config.node.channel_archiver {
+        let e = Error::with_msg_no_trace("archapp not built");
+        return Err((e, netout))?;
+    } else if let Some(_) = &node_config.node.archiver_appliance {
+        let e = Error::with_msg_no_trace("archapp not built");
+        return Err((e, netout))?;
+    } else {
+        let stream = match evq.agg_kind {
+            AggKind::EventBlobs => match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {
+                Ok(_stream) => {
+                    let e = Error::with_msg_no_trace("TODO make_event_blobs_pipe");
+                    return Err((e, netout))?;
+                }
+                Err(e) => return Err((e, netout))?,
+            },
+            _ => match disk::raw::conn::make_event_pipe(&evq, node_config).await {
+                Ok(j) => j,
+                Err(e) => return Err((e, netout))?,
+            },
+        };
+        stream
+    };
     let mut buf_len_histo = HistoLog2::new(5);
     while let Some(item) = p1.next().await {
         let item = item.make_frame();