WIP on getting something binned

disk/src/agg.rs

@@ -1,3 +1,13 @@
+#[allow(unused_imports)]
+use tracing::{error, warn, info, debug, trace};
+use err::Error;
+use std::task::{Context, Poll};
+use std::pin::Pin;
+use crate::EventFull;
+use futures_core::Stream;
+use futures_util::{pin_mut, StreamExt, future::ready};
+use netpod::ScalarType;
+
 pub trait AggregatorTdim {
     type OutputValue: AggregatableXdim1Bin + AggregatableTdim;
 }
@@ -36,22 +46,42 @@ pub struct ValuesDim1 {
     values: Vec<Vec<f32>>,
 }
 
+impl std::fmt::Debug for ValuesDim1 {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "count {}  tsA {:?}  tsB {:?}", self.tss.len(), self.tss.first(), self.tss.last())
+    }
+}
+
 impl AggregatableXdim1Bin for ValuesDim1 {
     type Output = MinMaxAvgScalarEventBatch;
+
     fn into_agg(self) -> Self::Output {
-        todo!()
+        let mut ret = MinMaxAvgScalarEventBatch {
+            tss: Vec::with_capacity(self.tss.len()),
+            mins: Vec::with_capacity(self.tss.len()),
+            maxs: Vec::with_capacity(self.tss.len()),
+            avgs: Vec::with_capacity(self.tss.len()),
+        };
+        // TODO do the actual binning
+        ret
     }
+
 }
 
 
 pub struct MinMaxAvgScalarEventBatch {
-    ts1s: Vec<u64>,
-    ts2s: Vec<u64>,
+    tss: Vec<u64>,
     mins: Vec<f32>,
     maxs: Vec<f32>,
     avgs: Vec<f32>,
 }
 
+impl std::fmt::Debug for MinMaxAvgScalarEventBatch {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "MinMaxAvgScalarEventBatch  count {}", self.tss.len())
+    }
+}
+
 impl AggregatableXdim1Bin for MinMaxAvgScalarEventBatch {
     type Output = MinMaxAvgScalarEventBatch;
     fn into_agg(self) -> Self::Output {
@@ -124,8 +154,139 @@ impl AggregatableXdim1Bin for MinMaxAvgScalarBinSingle {
     }
 }
 
+
+
+
+pub struct Dim1F32Stream<S>
+where S: Stream<Item=Result<EventFull, Error>>
+{
+    inp: S,
+}
+
+impl<S> Stream for Dim1F32Stream<S>
+where S: Stream<Item=Result<EventFull, Error>> + Unpin
+{
+    type Item = Result<ValuesDim1, Error>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        use Poll::*;
+        match self.inp.poll_next_unpin(cx) {
+            Ready(Some(Ok(k))) => {
+                let mut ret = ValuesDim1 {
+                    tss: vec![],
+                    values: vec![],
+                };
+                use ScalarType::*;
+                for i1 in 0..k.tss.len() {
+                    // TODO iterate sibling arrays after single bounds check
+                    let ty = &k.scalar_types[i1];
+                    let decomp = k.decomps[i1].as_ref().unwrap();
+                    match ty {
+                        F64 => {
+                            // do the conversion
+                            let n1 = decomp.len();
+                            assert!(n1 % ty.bytes() as usize == 0);
+                            let ele_count = n1 / ty.bytes() as usize;
+                            let mut j = Vec::with_capacity(ele_count);
+                            // this is safe for ints and floats
+                            unsafe { j.set_len(ele_count); }
+                            let mut p1 = 0;
+                            for i1 in 0..ele_count {
+                                unsafe {
+                                    j[i1] = std::mem::transmute_copy::<_, f64>(&decomp[p1]) as f32;
+                                    p1 += 8;
+                                }
+                            }
+                            ret.tss.push(k.tss[i1]);
+                            ret.values.push(j);
+                        },
+                        _ => todo!()
+                    }
+                }
+                Ready(Some(Ok(ret)))
+            }
+            Ready(Some(Err(e))) => Ready(Some(Err(e))),
+            Ready(None) => Ready(None),
+            Pending => Pending,
+        }
+    }
+
+}
+
+pub trait IntoDim1F32Stream {
+    fn into_dim_1_f32_stream(self) -> Dim1F32Stream<Self>
+        where Self: Sized,
+              Self: Stream<Item=Result<EventFull, Error>>;
+}
+
+impl<T> IntoDim1F32Stream for T
+    where T: Stream<Item=Result<EventFull, Error>>
+{
+
+    fn into_dim_1_f32_stream(self) -> Dim1F32Stream<T>
+    {
+        Dim1F32Stream {
+            inp: self,
+        }
+    }
+
+}
+
+
+pub trait IntoBinnedXBins1<I: AggregatableXdim1Bin> {
+    type StreamOut;
+    fn into_binned_x_bins_1(self) -> Self::StreamOut where Self: Stream<Item=Result<I, Error>>;
+}
+
+impl<T, I: AggregatableXdim1Bin> IntoBinnedXBins1<I> for T where T: Stream<Item=Result<I, Error>> + Unpin {
+    type StreamOut = IntoBinnedXBins1DefaultStream<T, I>;
+
+    fn into_binned_x_bins_1(self) -> Self::StreamOut {
+        IntoBinnedXBins1DefaultStream {
+            inp: self,
+        }
+    }
+
+}
+
+pub struct IntoBinnedXBins1DefaultStream<S, I> where S: Stream<Item=Result<I, Error>> + Unpin, I: AggregatableXdim1Bin {
+    inp: S,
+}
+
+impl<S, I> Stream for IntoBinnedXBins1DefaultStream<S, I> where S: Stream<Item=Result<I, Error>> + Unpin, I: AggregatableXdim1Bin {
+    type Item = Result<MinMaxAvgScalarEventBatch, Error>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        use Poll::*;
+        match self.inp.poll_next_unpin(cx) {
+            Ready(Some(Ok(k))) => {
+                let ret = MinMaxAvgScalarEventBatch {
+                    // TODO fill in the details
+                    tss: vec![],
+                    mins: vec![],
+                    maxs: vec![],
+                    avgs: vec![],
+                };
+                Ready(Some(Ok(ret)))
+            }
+            Ready(Some(Err(e))) => Ready(Some(Err(e))),
+            Ready(None) => Ready(None),
+            Pending => Pending,
+        }
+    }
+
+}
+
+
+
+
+
 #[test]
 fn agg_x_dim_1() {
+    crate::run(async { agg_x_dim_1_inner().await; Ok(()) }).unwrap();
+}
+
+async fn agg_x_dim_1_inner() {
     let vals = ValuesDim1 {
         tss: vec![0, 1, 2, 3],
         values: vec![
@@ -149,5 +310,39 @@ fn agg_x_dim_1() {
     How do I want to drive the system?
     If I write the T-binner as a Stream, then I also need to pass it the input!
     Meaning, I need to pass the Stream which produces the actual numbers from disk.
+
+    readchannel()  -> Stream of timestamped byte blobs
+    .to_f32()  -> Stream ?    indirection to branch on the underlying shape
+    .agg_x_bins_1()  -> Stream ?    can I keep it at the single indirection on the top level?
     */
+    let query = netpod::AggQuerySingleChannel {
+        ksprefix: "daq_swissfel".into(),
+        keyspace: 3,
+        channel: netpod::Channel {
+            name: "S10BC01-DBAM070:BAM_CH1_NORM".into(),
+            backend: "sf-databuffer".into(),
+        },
+        timebin: 18721,
+        tb_file_count: 1,
+        split: 12,
+        tbsize: 1000 * 60 * 60 * 24,
+        buffer_size: 1024 * 4,
+    };
+    let fut1 = crate::EventBlobsComplete::new(&query)
+    .into_dim_1_f32_stream()
+    .take(10)
+    .map(|q| {
+        if let Ok(ref k) = q {
+            info!("vals: {:?}", k);
+        }
+        q
+    })
+    .into_binned_x_bins_1()
+    .map(|k| {
+        let k = k.unwrap();
+        info!("after X binning  {:?}", k);
+        k
+    })
+    .for_each(|k| ready(()));
+    fut1.await;
 }