Emit events in better batch sizes

items/src/eventvalues.rs

@@ -2,7 +2,7 @@ use crate::minmaxavgbins::MinMaxAvgBins;
 use crate::numops::NumOps;
 use crate::streams::{Collectable, Collector};
 use crate::{
-    ts_offs_from_abs, Appendable, EventAppendable, FilterFittingInside, Fits, FitsInside, PushableIndex,
+    ts_offs_from_abs, Appendable, ByteEstimate, EventAppendable, FilterFittingInside, Fits, FitsInside, PushableIndex,
     RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, TimeBinnableType, TimeBinnableTypeAggregator,
     WithLen, WithTimestamps,
 };
@@ -13,12 +13,12 @@ use serde::{Deserialize, Serialize};
 use std::fmt;
 use tokio::fs::File;
 
-// TODO add pulse.
+// TODO add pulse. Is this even used??
 // TODO change name, it's not only about values, but more like batch of whole events.
 #[derive(Serialize, Deserialize)]
-pub struct EventValues<VT> {
+pub struct EventValues<NTY> {
     pub tss: Vec<u64>,
-    pub values: Vec<VT>,
+    pub values: Vec<NTY>,
 }
 
 impl<NTY> SitemtyFrameType for EventValues<NTY>
@@ -28,7 +28,7 @@ where
     const FRAME_TYPE_ID: u32 = crate::EVENT_VALUES_FRAME_TYPE_ID + NTY::SUB;
 }
 
-impl<VT> EventValues<VT> {
+impl<NTY> EventValues<NTY> {
     pub fn empty() -> Self {
         Self {
             tss: vec![],
@@ -37,11 +37,11 @@ impl<VT> EventValues<VT> {
     }
 }
 
-impl<VT> fmt::Debug for EventValues<VT>
+impl<NTY> fmt::Debug for EventValues<NTY>
 where
-    VT: fmt::Debug,
+    NTY: fmt::Debug,
 {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         write!(
             fmt,
             "count {}  ts {:?} .. {:?}  vals {:?} .. {:?}",
@@ -54,19 +54,39 @@ where
     }
 }
 
-impl<VT> WithLen for EventValues<VT> {
+impl<NTY> WithLen for EventValues<NTY>
+where
+    NTY: NumOps,
+{
     fn len(&self) -> usize {
         self.tss.len()
     }
 }
 
-impl<VT> WithTimestamps for EventValues<VT> {
+impl<NTY> WithTimestamps for EventValues<NTY>
+where
+    NTY: NumOps,
+{
     fn ts(&self, ix: usize) -> u64 {
         self.tss[ix]
     }
 }
 
-impl<VT> RangeOverlapInfo for EventValues<VT> {
+impl<NTY> ByteEstimate for EventValues<NTY>
+where
+    NTY: NumOps,
+{
+    fn byte_estimate(&self) -> u64 {
+        if self.tss.len() == 0 {
+            0
+        } else {
+            // TODO improve via a const fn on NTY
+            self.tss.len() as u64 * 16
+        }
+    }
+}
+
+impl<NTY> RangeOverlapInfo for EventValues<NTY> {
     fn ends_before(&self, range: NanoRange) -> bool {
         match self.tss.last() {
             Some(&ts) => ts < range.beg,
@@ -89,7 +109,7 @@ impl<VT> RangeOverlapInfo for EventValues<VT> {
     }
 }
 
-impl<VT> FitsInside for EventValues<VT> {
+impl<NTY> FitsInside for EventValues<NTY> {
     fn fits_inside(&self, range: NanoRange) -> Fits {
         if self.tss.is_empty() {
             Fits::Empty
@@ -113,7 +133,7 @@ impl<VT> FitsInside for EventValues<VT> {
     }
 }
 
-impl<VT> FilterFittingInside for EventValues<VT> {
+impl<NTY> FilterFittingInside for EventValues<NTY> {
     fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
         match self.fits_inside(fit_range) {
             Fits::Inside | Fits::PartlyGreater | Fits::PartlyLower | Fits::PartlyLowerAndGreater => Some(self),

items/src/lib.rs

@@ -238,7 +238,7 @@ where
 
 pub trait EventsNodeProcessor: Send + Unpin {
     type Input;
-    type Output: Send + Unpin + DeserializeOwned + WithTimestamps + TimeBinnableType;
+    type Output: Send + Unpin + DeserializeOwned + WithTimestamps + TimeBinnableType + ByteEstimate;
     fn create(shape: Shape, agg_kind: AggKind) -> Self;
     fn process(&self, inp: Self::Input) -> Self::Output;
 }
@@ -279,6 +279,10 @@ pub trait WithTimestamps {
     fn ts(&self, ix: usize) -> u64;
 }
 
+pub trait ByteEstimate {
+    fn byte_estimate(&self) -> u64;
+}
+
 pub trait RangeOverlapInfo {
     fn ends_before(&self, range: NanoRange) -> bool;
     fn ends_after(&self, range: NanoRange) -> bool;

items/src/waveevents.rs

@@ -3,8 +3,8 @@ use crate::numops::NumOps;
 use crate::xbinnedscalarevents::XBinnedScalarEvents;
 use crate::xbinnedwaveevents::XBinnedWaveEvents;
 use crate::{
-    Appendable, EventAppendable, EventsNodeProcessor, FilterFittingInside, Fits, FitsInside, PushableIndex,
-    RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType,
+    Appendable, ByteEstimate, EventAppendable, EventsNodeProcessor, FilterFittingInside, Fits, FitsInside,
+    PushableIndex, RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType,
     TimeBinnableTypeAggregator, WithLen, WithTimestamps,
 };
 use err::Error;
@@ -48,6 +48,17 @@ impl<NTY> WithTimestamps for WaveEvents<NTY> {
     }
 }
 
+impl<NTY> ByteEstimate for WaveEvents<NTY> {
+    fn byte_estimate(&self) -> u64 {
+        if self.tss.len() == 0 {
+            0
+        } else {
+            // TODO improve via a const fn on NTY
+            self.tss.len() as u64 * 8 * self.vals[0].len() as u64
+        }
+    }
+}
+
 impl<NTY> RangeOverlapInfo for WaveEvents<NTY> {
     fn ends_before(&self, range: NanoRange) -> bool {
         match self.tss.last() {

items/src/xbinnedscalarevents.rs

@@ -2,8 +2,9 @@ use crate::minmaxavgbins::MinMaxAvgBins;
 use crate::numops::NumOps;
 use crate::streams::{Collectable, Collector};
 use crate::{
-    ts_offs_from_abs, Appendable, FilterFittingInside, Fits, FitsInside, PushableIndex, RangeOverlapInfo, ReadPbv,
-    ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, WithLen, WithTimestamps,
+    ts_offs_from_abs, Appendable, ByteEstimate, FilterFittingInside, Fits, FitsInside, PushableIndex, RangeOverlapInfo,
+    ReadPbv, ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, WithLen,
+    WithTimestamps,
 };
 use err::Error;
 use netpod::timeunits::SEC;
@@ -50,6 +51,17 @@ impl<NTY> WithTimestamps for XBinnedScalarEvents<NTY> {
     }
 }
 
+impl<NTY> ByteEstimate for XBinnedScalarEvents<NTY> {
+    fn byte_estimate(&self) -> u64 {
+        if self.tss.len() == 0 {
+            0
+        } else {
+            // TODO improve via a const fn on NTY
+            self.tss.len() as u64 * 28
+        }
+    }
+}
+
 impl<NTY> RangeOverlapInfo for XBinnedScalarEvents<NTY> {
     fn ends_before(&self, range: NanoRange) -> bool {
         match self.tss.last() {

items/src/xbinnedwaveevents.rs

@@ -2,8 +2,8 @@ use crate::minmaxavgwavebins::MinMaxAvgWaveBins;
 use crate::numops::NumOps;
 use crate::streams::{Collectable, Collector};
 use crate::{
-    Appendable, FilterFittingInside, Fits, FitsInside, PushableIndex, RangeOverlapInfo, ReadPbv, ReadableFromFile,
-    SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, WithLen, WithTimestamps,
+    Appendable, ByteEstimate, FilterFittingInside, Fits, FitsInside, PushableIndex, RangeOverlapInfo, ReadPbv,
+    ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, WithLen, WithTimestamps,
 };
 use err::Error;
 use netpod::log::*;
@@ -51,6 +51,17 @@ impl<NTY> WithTimestamps for XBinnedWaveEvents<NTY> {
     }
 }
 
+impl<NTY> ByteEstimate for XBinnedWaveEvents<NTY> {
+    fn byte_estimate(&self) -> u64 {
+        if self.tss.len() == 0 {
+            0
+        } else {
+            // TODO improve via a const fn on NTY
+            self.tss.len() as u64 * 20 * self.avgs[0].len() as u64
+        }
+    }
+}
+
 impl<NTY> RangeOverlapInfo for XBinnedWaveEvents<NTY> {
     fn ends_before(&self, range: NanoRange) -> bool {
         match self.tss.last() {