use crate::minmaxavgbins::MinMaxAvgBins; use crate::numops::NumOps; use crate::streams::{Collectable, Collector}; use crate::{ ts_offs_from_abs, Appendable, FilterFittingInside, Fits, FitsInside, PushableIndex, RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, WithLen, WithTimestamps, }; use err::Error; use netpod::timeunits::SEC; use netpod::NanoRange; use serde::{Deserialize, Serialize}; use tokio::fs::File; // TODO rename Scalar -> Dim0 #[derive(Debug, Serialize, Deserialize)] pub struct XBinnedScalarEvents { pub tss: Vec, pub mins: Vec, pub maxs: Vec, pub avgs: Vec, } impl SitemtyFrameType for XBinnedScalarEvents where NTY: SubFrId, { const FRAME_TYPE_ID: u32 = 0x600 + NTY::SUB; } impl XBinnedScalarEvents { pub fn empty() -> Self { Self { tss: vec![], mins: vec![], maxs: vec![], avgs: vec![], } } } impl WithLen for XBinnedScalarEvents { fn len(&self) -> usize { self.tss.len() } } impl WithTimestamps for XBinnedScalarEvents { fn ts(&self, ix: usize) -> u64 { self.tss[ix] } } impl RangeOverlapInfo for XBinnedScalarEvents { fn ends_before(&self, range: NanoRange) -> bool { match self.tss.last() { Some(&ts) => ts < range.beg, None => true, } } fn ends_after(&self, range: NanoRange) -> bool { match self.tss.last() { Some(&ts) => ts >= range.end, None => panic!(), } } fn starts_after(&self, range: NanoRange) -> bool { match self.tss.first() { Some(&ts) => ts >= range.end, None => panic!(), } } } impl FitsInside for XBinnedScalarEvents { fn fits_inside(&self, range: NanoRange) -> Fits { if self.tss.is_empty() { Fits::Empty } else { let t1 = *self.tss.first().unwrap(); let t2 = *self.tss.last().unwrap(); if t2 < range.beg { Fits::Lower } else if t1 > range.end { Fits::Greater } else if t1 < range.beg && t2 > range.end { Fits::PartlyLowerAndGreater } else if t1 < range.beg { Fits::PartlyLower } else if t2 > range.end { Fits::PartlyGreater } else { Fits::Inside } } } } impl FilterFittingInside for XBinnedScalarEvents { fn filter_fitting_inside(self, fit_range: NanoRange) -> Option { match self.fits_inside(fit_range) { Fits::Inside | Fits::PartlyGreater | Fits::PartlyLower | Fits::PartlyLowerAndGreater => Some(self), _ => None, } } } impl PushableIndex for XBinnedScalarEvents where NTY: NumOps, { fn push_index(&mut self, src: &Self, ix: usize) { self.tss.push(src.tss[ix]); self.mins.push(src.mins[ix]); self.maxs.push(src.maxs[ix]); self.avgs.push(src.avgs[ix]); } } impl Appendable for XBinnedScalarEvents where NTY: NumOps, { fn empty() -> Self { Self::empty() } fn append(&mut self, src: &Self) { self.tss.extend_from_slice(&src.tss); self.mins.extend_from_slice(&src.mins); self.maxs.extend_from_slice(&src.maxs); self.avgs.extend_from_slice(&src.avgs); } } impl ReadableFromFile for XBinnedScalarEvents where NTY: NumOps, { fn read_from_file(_file: File) -> Result, Error> { // TODO refactor types such that this impl is not needed. panic!() } fn from_buf(_buf: &[u8]) -> Result { panic!() } } impl TimeBinnableType for XBinnedScalarEvents where NTY: NumOps, { type Output = MinMaxAvgBins; type Aggregator = XBinnedScalarEventsAggregator; fn aggregator(range: NanoRange, _x_bin_count: usize, do_time_weight: bool) -> Self::Aggregator { // TODO remove output if range.delta() > SEC * 0 { netpod::log::info!( "TimeBinnableType for XBinnedScalarEvents aggregator() range {:?}", range ); } Self::Aggregator::new(range, do_time_weight) } } pub struct XBinnedScalarEventsAggregator where NTY: NumOps, { range: NanoRange, count: u64, min: Option, max: Option, sumc: u64, sum: f32, int_ts: u64, last_ts: u64, last_avg: Option, last_min: Option, last_max: Option, do_time_weight: bool, } impl XBinnedScalarEventsAggregator where NTY: NumOps, { pub fn new(range: NanoRange, do_time_weight: bool) -> Self { let int_ts = range.beg; Self { range, count: 0, min: None, max: None, sumc: 0, sum: 0f32, int_ts, last_ts: 0, last_avg: None, last_min: None, last_max: None, do_time_weight, } } fn apply_min_max(&mut self, min: NTY, max: NTY) { self.min = match self.min { None => Some(min), Some(cmin) => { if min < cmin { Some(min) } else { Some(cmin) } } }; self.max = match self.max { None => Some(max), Some(cmax) => { if max > cmax { Some(max) } else { Some(cmax) } } }; } fn apply_event_unweight(&mut self, avg: f32, min: NTY, max: NTY) { self.apply_min_max(min, max); let vf = avg; if vf.is_nan() { } else { self.sum += vf; self.sumc += 1; } } fn apply_event_time_weight(&mut self, ts: u64, avg: Option, min: Option, max: Option) { if let Some(v) = self.last_avg { self.apply_min_max(min.unwrap(), max.unwrap()); let w = if self.do_time_weight { (ts - self.int_ts) as f32 * 1e-9 } else { 1. }; let vf = v; if vf.is_nan() { } else { self.sum += vf * w; self.sumc += 1; } self.int_ts = ts; } self.last_ts = ts; self.last_avg = avg; self.last_min = min; self.last_max = max; } fn ingest_unweight(&mut self, item: &XBinnedScalarEvents) { for i1 in 0..item.tss.len() { let ts = item.tss[i1]; let avg = item.avgs[i1]; let min = item.mins[i1]; let max = item.maxs[i1]; if ts < self.range.beg { } else if ts >= self.range.end { } else { self.count += 1; self.apply_event_unweight(avg, min, max); } } } fn ingest_time_weight(&mut self, item: &XBinnedScalarEvents) { for i1 in 0..item.tss.len() { let ts = item.tss[i1]; let avg = item.avgs[i1]; let min = item.mins[i1]; let max = item.maxs[i1]; if ts < self.int_ts { self.last_ts = ts; self.last_avg = Some(avg); self.last_min = Some(min); self.last_max = Some(max); } else if ts >= self.range.end { return; } else { self.count += 1; self.apply_event_time_weight(ts, Some(avg), Some(min), Some(max)); } } } fn result_reset_unweight(&mut self, range: NanoRange, _expand: bool) -> MinMaxAvgBins { let avg = if self.sumc == 0 { None } else { Some(self.sum / self.sumc as f32) }; let ret = MinMaxAvgBins { ts1s: vec![self.range.beg], ts2s: vec![self.range.end], counts: vec![self.count], mins: vec![self.min], maxs: vec![self.max], avgs: vec![avg], }; self.range = range; self.count = 0; self.min = None; self.max = None; self.sum = 0f32; self.sumc = 0; ret } fn result_reset_time_weight(&mut self, range: NanoRange, expand: bool) -> MinMaxAvgBins { if expand { self.apply_event_time_weight(self.range.end, self.last_avg, self.last_min, self.last_max); } let avg = { let sc = self.range.delta() as f32 * 1e-9; Some(self.sum / sc) }; let ret = MinMaxAvgBins { ts1s: vec![self.range.beg], ts2s: vec![self.range.end], counts: vec![self.count], mins: vec![self.min], maxs: vec![self.max], avgs: vec![avg], }; self.range = range; self.count = 0; self.min = None; self.max = None; self.sum = 0f32; self.sumc = 0; ret } } impl TimeBinnableTypeAggregator for XBinnedScalarEventsAggregator where NTY: NumOps, { type Input = XBinnedScalarEvents; type Output = MinMaxAvgBins; fn range(&self) -> &NanoRange { &self.range } fn ingest(&mut self, item: &Self::Input) { if self.do_time_weight { self.ingest_time_weight(item) } else { self.ingest_unweight(item) } } fn result_reset(&mut self, range: NanoRange, expand: bool) -> Self::Output { if self.do_time_weight { self.result_reset_time_weight(range, expand) } else { self.result_reset_unweight(range, expand) } } } #[derive(Serialize, Deserialize)] pub struct XBinnedScalarEventsCollectedResult { #[serde(rename = "tsAnchor")] ts_anchor_sec: u64, #[serde(rename = "tsMs")] ts_off_ms: Vec, #[serde(rename = "tsNs")] ts_off_ns: Vec, mins: Vec, maxs: Vec, avgs: Vec, #[serde(skip_serializing_if = "crate::bool_is_false", rename = "finalisedRange")] finalised_range: bool, #[serde(skip_serializing_if = "crate::bool_is_false", rename = "timedOut")] timed_out: bool, } pub struct XBinnedScalarEventsCollector { vals: XBinnedScalarEvents, finalised_range: bool, timed_out: bool, #[allow(dead_code)] bin_count_exp: u32, } impl XBinnedScalarEventsCollector { pub fn new(bin_count_exp: u32) -> Self { Self { finalised_range: false, timed_out: false, vals: XBinnedScalarEvents::empty(), bin_count_exp, } } } impl WithLen for XBinnedScalarEventsCollector { fn len(&self) -> usize { self.vals.tss.len() } } impl Collector for XBinnedScalarEventsCollector where NTY: NumOps, { type Input = XBinnedScalarEvents; type Output = XBinnedScalarEventsCollectedResult; fn ingest(&mut self, src: &Self::Input) { self.vals.append(src); } fn set_range_complete(&mut self) { self.finalised_range = true; } fn set_timed_out(&mut self) { self.timed_out = true; } fn result(self) -> Result { let tst = ts_offs_from_abs(&self.vals.tss); let ret = Self::Output { ts_anchor_sec: tst.0, ts_off_ms: tst.1, ts_off_ns: tst.2, mins: self.vals.mins, maxs: self.vals.maxs, avgs: self.vals.avgs, finalised_range: self.finalised_range, timed_out: self.timed_out, }; Ok(ret) } } impl Collectable for XBinnedScalarEvents where NTY: NumOps, { type Collector = XBinnedScalarEventsCollector; fn new_collector(bin_count_exp: u32) -> Self::Collector { Self::Collector::new(bin_count_exp) } }