daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor and unify

Browse Source
This commit is contained in:
Dominik Werder
2022-07-26 11:29:14 +02:00
parent d0a7240934
commit f195a941f1
14 changed files with 952 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
items/src/binsdim0.rs
View File

@@ -367,6 +367,8 @@ pub struct MinMaxAvgDim0BinsAggregator<NTY> {
count: u64,
min: NTY,
max: NTY,
// Carry over to next bin:
avg: f32,
sumc: u64,
sum: f32,
}
@@ -378,6 +380,7 @@ impl<NTY: NumOps> MinMaxAvgDim0BinsAggregator<NTY> {
count: 0,
min: NTY::zero(),
max: NTY::zero(),
avg: 0.,
sumc: 0,
sum: 0f32,
}
@@ -401,14 +404,17 @@ where
} else if item.ts2s[i1] <= self.range.beg {
} else if item.ts1s[i1] >= self.range.end {
} else {
if item.mins[i1].as_prim_f32() < 1. {
info!("small bin min {:?} counts {}", item.mins[i1], item.counts[i1]);
}
if self.count == 0 {
self.min = item.mins[i1].clone();
self.max = item.maxs[i1].clone();
} else {
if item.mins[i1] < self.min {
if self.min > item.mins[i1] {
self.min = item.mins[i1].clone();
}
if item.maxs[i1] > self.max {
if self.max < item.maxs[i1] {
self.max = item.maxs[i1].clone();
}
}
@@ -420,23 +426,19 @@ where
}
fn result_reset(&mut self, range: NanoRange, _expand: bool) -> Self::Output {
let avg = if self.sumc == 0 {
0f32
} else {
self.sum / self.sumc as f32
};
if self.sumc > 0 {
self.avg = self.sum / self.sumc as f32;
}
let ret = Self::Output {
ts1s: vec![self.range.beg],
ts2s: vec![self.range.end],
counts: vec![self.count],
mins: vec![self.min.clone()],
maxs: vec![self.max.clone()],
avgs: vec![avg],
avgs: vec![self.avg],
};
self.count = 0;
self.min = NTY::zero();
self.max = NTY::zero();
self.range = range;
self.count = 0;
self.sum = 0f32;
self.sumc = 0;
ret
@@ -459,86 +461,35 @@ impl<NTY: NumOps + 'static> TimeBinnableDyn for MinMaxAvgDim0Bins<NTY> {
pub struct MinMaxAvgDim0BinsTimeBinner<NTY: NumOps> {
edges: VecDeque<u64>,
do_time_weight: bool,
range: NanoRange,
agg: Option<MinMaxAvgDim0BinsAggregator<NTY>>,
ready: Option<<MinMaxAvgDim0BinsAggregator<NTY> as TimeBinnableTypeAggregator>::Output>,
}
impl<NTY: NumOps> MinMaxAvgDim0BinsTimeBinner<NTY> {
fn new(edges: VecDeque<u64>, do_time_weight: bool) -> Self {
let range = if edges.len() >= 2 {
NanoRange {
beg: edges[0],
end: edges[1],
}
} else {
// Using a dummy for this case.
NanoRange { beg: 1, end: 2 }
};
Self {
edges,
do_time_weight,
range,
agg: None,
ready: None,
}
}
// Move the bin from the current aggregator (if any) to our output collection,
// and step forward in our bin list.
fn cycle(&mut self) {
eprintln!("cycle");
// TODO where to take expand from? Is it still required after all?
let expand = true;
let have_next_bin = self.edges.len() >= 3;
let range_next = if have_next_bin {
NanoRange {
beg: self.edges[1],
end: self.edges[2],
}
fn next_bin_range(&mut self) -> Option<NanoRange> {
if self.edges.len() >= 2 {
let ret = NanoRange {
beg: self.edges[0],
end: self.edges[1],
};
self.edges.pop_front();
Some(ret)
} else {
// Using a dummy for this case.
NanoRange { beg: 1, end: 2 }
};
if let Some(agg) = self.agg.as_mut() {
eprintln!("cycle: use existing agg: {:?}", agg.range);
let mut h = agg.result_reset(range_next.clone(), expand);
match self.ready.as_mut() {
Some(fin) => {
fin.append(&mut h);
}
None => {
self.ready = Some(h);
}
}
} else if have_next_bin {
eprintln!("cycle: append a zero bin");
let mut h = MinMaxAvgDim0Bins::<NTY>::empty();
h.append_zero(self.range.beg, self.range.end);
match self.ready.as_mut() {
Some(fin) => {
fin.append(&mut h);
}
None => {
self.ready = Some(h);
}
}
} else {
eprintln!("cycle: no more next bin");
}
self.range = range_next;
self.edges.pop_front();
if !have_next_bin {
self.agg = None;
None
}
}
}
impl<NTY: NumOps + 'static> TimeBinnerDyn for MinMaxAvgDim0BinsTimeBinner<NTY> {
fn cycle(&mut self) {
Self::cycle(self)
}
fn ingest(&mut self, item: &dyn TimeBinnableDyn) {
const SELF: &str = "MinMaxAvgDim0BinsTimeBinner";
if item.len() == 0 {
@@ -552,37 +503,48 @@ impl<NTY: NumOps + 'static> TimeBinnerDyn for MinMaxAvgDim0BinsTimeBinner<NTY> {
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(self.range.clone()) {
while item.starts_after(NanoRange {
beg: 0,
end: self.edges[1],
}) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} no more bin in edges B");
return;
}
}
if item.ends_before(self.range.clone()) {
if item.ends_before(NanoRange {
beg: self.edges[0],
end: u64::MAX,
}) {
return;
} else {
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} edge list exhausted");
return;
} else {
if self.agg.is_none() {
let agg = if let Some(agg) = self.agg.as_mut() {
agg
} else {
self.agg = Some(MinMaxAvgDim0BinsAggregator::new(
self.range.clone(),
// We know here that we have enough edges for another bin.
// and `next_bin_range` will pop the first edge.
self.next_bin_range().unwrap(),
self.do_time_weight,
));
}
let agg = self.agg.as_mut().unwrap();
if let Some(item) =
item.as_any()
.downcast_ref::<<MinMaxAvgDim0BinsAggregator<NTY> as TimeBinnableTypeAggregator>::Input>()
self.agg.as_mut().unwrap()
};
if let Some(item) = item
.as_any()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_ref::<<MinMaxAvgDim0BinsAggregator<NTY> as TimeBinnableTypeAggregator>::Input>()
{
agg.ingest(item);
} else {
let tyid_item = std::any::Any::type_id(item.as_any());
error!("not correct item type {:?}", tyid_item);
};
if item.ends_after(self.range.clone()) {
if item.ends_after(agg.range().clone()) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} no more bin in edges C");
@@ -609,6 +571,53 @@ impl<NTY: NumOps + 'static> TimeBinnerDyn for MinMaxAvgDim0BinsTimeBinner<NTY> {
None => None,
}
}
// TODO there is too much common code between implementors:
fn push_in_progress(&mut self, push_empty: bool) {
// TODO expand should be derived from AggKind. Is it still required after all?
let expand = true;
if let Some(agg) = self.agg.as_mut() {
let dummy_range = NanoRange { beg: 4, end: 5 };
let mut bins = agg.result_reset(dummy_range, expand);
self.agg = None;
assert_eq!(bins.len(), 1);
if push_empty || bins.counts[0] != 0 {
match self.ready.as_mut() {
Some(ready) => {
ready.append(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
}
}
}
// TODO there is too much common code between implementors:
fn cycle(&mut self) {
let n = self.bins_ready_count();
self.push_in_progress(true);
if self.bins_ready_count() == n {
if let Some(range) = self.next_bin_range() {
let mut bins = MinMaxAvgDim0Bins::<NTY>::empty();
bins.append_zero(range.beg, range.end);
match self.ready.as_mut() {
Some(ready) => {
ready.append(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
if self.bins_ready_count() <= n {
error!("failed to push a zero bin");
}
} else {
warn!("cycle: no in-progress bin pushed, but also no more bin to add as zero-bin");
}
}
}
}
impl<NTY: NumOps> TimeBinned for MinMaxAvgDim0Bins<NTY> {
@@ -635,4 +644,22 @@ impl<NTY: NumOps> TimeBinned for MinMaxAvgDim0Bins<NTY> {
fn avgs(&self) -> Vec<f32> {
self.avgs.clone()
}
fn validate(&self) -> Result<(), String> {
use std::fmt::Write;
let mut msg = String::new();
if self.ts1s.len() != self.ts2s.len() {
write!(&mut msg, "ts1s ≠ ts2s\n").unwrap();
}
for (i, ((count, min), max)) in self.counts.iter().zip(&self.mins).zip(&self.maxs).enumerate() {
if min.as_prim_f32() < 1. && *count != 0 {
write!(&mut msg, "i {} count {} min {:?} max {:?}\n", i, count, min, max).unwrap();
}
}
if msg.is_empty() {
Ok(())
} else {
Err(msg)
}
}
}

4
items/src/binsdim1.rs
View File

@@ -594,4 +594,8 @@ impl<NTY: NumOps> TimeBinned for MinMaxAvgDim1Bins<NTY> {
fn maxs(&self) -> Vec<f32> {
err::todoval()
}
fn validate(&self) -> Result<(), String> {
err::todoval()
}
}

38
items/src/frame.rs
View File

@@ -1,7 +1,7 @@
use crate::inmem::InMemoryFrame;
use crate::{
FrameType, FrameTypeStatic, ERROR_FRAME_TYPE_ID, INMEM_FRAME_ENCID, INMEM_FRAME_HEAD, INMEM_FRAME_MAGIC,
TERM_FRAME_TYPE_ID,
FrameType, FrameTypeStatic, Sitemty, StreamItem, ERROR_FRAME_TYPE_ID, INMEM_FRAME_ENCID, INMEM_FRAME_HEAD,
INMEM_FRAME_MAGIC, NON_DATA_FRAME_TYPE_ID, TERM_FRAME_TYPE_ID,
};
use bytes::{BufMut, BytesMut};
use err::Error;
@@ -144,6 +144,40 @@ where
}
};
Ok(T::from_error(k))
} else if frame.tyid() == NON_DATA_FRAME_TYPE_ID {
error!("TODO NON_DATA_FRAME_TYPE_ID");
type TT = Sitemty<crate::scalarevents::ScalarEvents<u32>>;
let _k: TT = match bincode::deserialize::<TT>(frame.buf()) {
Ok(item) => match item {
Ok(StreamItem::DataItem(_)) => {
error!(
"ERROR bincode::deserialize len {} NON_DATA_FRAME_TYPE_ID but found Ok(StreamItem::DataItem)",
frame.buf().len()
);
let n = frame.buf().len().min(64);
let s = String::from_utf8_lossy(&frame.buf()[..n]);
error!("frame.buf as string: {:?}", s);
Err(Error::with_msg_no_trace("NON_DATA_FRAME_TYPE_ID decode error"))?
}
Ok(StreamItem::Log(k)) => Ok(StreamItem::Log(k)),
Ok(StreamItem::Stats(k)) => Ok(StreamItem::Stats(k)),
Err(e) => {
error!("decode_frame sees error: {e:?}");
Err(e)
}
},
Err(e) => {
error!(
"ERROR bincode::deserialize len {} ERROR_FRAME_TYPE_ID",
frame.buf().len()
);
let n = frame.buf().len().min(64);
let s = String::from_utf8_lossy(&frame.buf()[..n]);
error!("frame.buf as string: {:?}", s);
Err(e)?
}
};
Err(Error::with_msg_no_trace("TODO NON_DATA_FRAME_TYPE_ID"))
} else {
let tyid = T::FRAME_TYPE_ID;
if frame.tyid() != tyid {

59
items/src/items.rs
View File

@@ -525,6 +525,8 @@ pub trait TimeBinnableDynAggregator: Send {
/// Container of some form of events, for use as trait object.
pub trait EventsDyn: TimeBinnableDyn {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnableDyn;
fn verify(&self);
fn output_info(&self);
}
/// Data in time-binned form.
@@ -535,6 +537,7 @@ pub trait TimeBinned: TimeBinnableDyn {
fn mins(&self) -> Vec<f32>;
fn maxs(&self) -> Vec<f32>;
fn avgs(&self) -> Vec<f32>;
fn validate(&self) -> Result<(), String>;
}
impl WithLen for Box<dyn TimeBinned> {
@@ -662,6 +665,8 @@ pub trait TimeBinnableTypeAggregator: Send {
type Output: TimeBinnableType;
fn range(&self) -> &NanoRange;
fn ingest(&mut self, item: &Self::Input);
// TODO this API is too convoluted for a minimal performance gain: should separate `result` and `reset`
// or simply require to construct a new which is almost equally expensive.
fn result_reset(&mut self, range: NanoRange, expand: bool) -> Self::Output;
}
@@ -690,8 +695,12 @@ pub trait TimeBinnerDyn: Send {
fn bins_ready(&mut self) -> Option<Box<dyn TimeBinned>>;
fn ingest(&mut self, item: &dyn TimeBinnableDyn);
/// Caller indicates that there will be no more data for the current bin.
/// Implementor is expected to prepare processing the next bin.
/// If there is a bin in progress with non-zero count, push it to the result set.
/// With push_empty == true, a bin in progress is pushed even if it contains no counts.
fn push_in_progress(&mut self, push_empty: bool);
/// Implies `Self::push_in_progress` but in addition, pushes a zero-count bin if the call
/// to `push_in_progress` did not change the result count, as long as edges are left.
/// The next call to `Self::bins_ready_count` must return one higher count than before.
fn cycle(&mut self);
}
@@ -777,7 +786,7 @@ pub fn empty_binned_dyn(scalar_type: &ScalarType, shape: &Shape, agg_kind: &AggK
#[test]
fn bin_binned_01() {
use binsdim0::MinMaxAvgDim0Bins;
let edges = vec![SEC * 1000, SEC * 1010, SEC * 1020];
let edges = vec![SEC * 1000, SEC * 1010, SEC * 1020, SEC * 1030];
let inp0 = <MinMaxAvgDim0Bins<u32> as NewEmpty>::empty(Shape::Scalar);
let mut time_binner = inp0.time_binner_new(edges, true);
let inp1 = MinMaxAvgDim0Bins::<u32> {
@@ -791,12 +800,52 @@ fn bin_binned_01() {
assert_eq!(time_binner.bins_ready_count(), 0);
time_binner.ingest(&inp1);
assert_eq!(time_binner.bins_ready_count(), 1);
time_binner.cycle();
time_binner.push_in_progress(false);
assert_eq!(time_binner.bins_ready_count(), 2);
// From here on, pushing any more should not change the bin count:
time_binner.push_in_progress(false);
assert_eq!(time_binner.bins_ready_count(), 2);
// On the other hand, cycling should add one more zero-bin:
time_binner.cycle();
assert_eq!(time_binner.bins_ready_count(), 3);
time_binner.cycle();
assert_eq!(time_binner.bins_ready_count(), 3);
let bins = time_binner.bins_ready().expect("bins should be ready");
eprintln!("bins: {:?}", bins);
assert_eq!(time_binner.bins_ready_count(), 0);
assert_eq!(bins.counts(), &[1, 1, 0]);
// TODO use proper float-compare logic:
assert_eq!(bins.mins(), &[3., 4., 0.]);
assert_eq!(bins.maxs(), &[10., 9., 0.]);
assert_eq!(bins.avgs(), &[7., 6., 0.]);
}
#[test]
fn bin_binned_02() {
use binsdim0::MinMaxAvgDim0Bins;
let edges = vec![SEC * 1000, SEC * 1020];
let inp0 = <MinMaxAvgDim0Bins<u32> as NewEmpty>::empty(Shape::Scalar);
let mut time_binner = inp0.time_binner_new(edges, true);
let inp1 = MinMaxAvgDim0Bins::<u32> {
ts1s: vec![SEC * 1000, SEC * 1010],
ts2s: vec![SEC * 1010, SEC * 1020],
counts: vec![1, 1],
mins: vec![3, 4],
maxs: vec![10, 9],
avgs: vec![7., 6.],
};
assert_eq!(time_binner.bins_ready_count(), 0);
time_binner.ingest(&inp1);
assert_eq!(time_binner.bins_ready_count(), 0);
time_binner.cycle();
assert_eq!(time_binner.bins_ready_count(), 1);
time_binner.cycle();
//assert_eq!(time_binner.bins_ready_count(), 2);
let bins = time_binner.bins_ready().expect("bins should be ready");
eprintln!("bins: {:?}", bins);
assert_eq!(bins.counts().len(), 2);
assert_eq!(time_binner.bins_ready_count(), 0);
assert_eq!(bins.counts(), &[2]);
assert_eq!(bins.mins(), &[3.]);
assert_eq!(bins.maxs(), &[10.]);
assert_eq!(bins.avgs(), &[13. / 2.]);
}

335
items/src/scalarevents.rs
View File

@@ -369,9 +369,10 @@ pub struct EventValuesAggregator<NTY> {
impl<NTY> Drop for EventValuesAggregator<NTY> {
fn drop(&mut self) {
// TODO collect as stats for the request context:
warn!(
trace!(
"taken {} ignored {}",
self.events_taken_count, self.events_ignored_count
self.events_taken_count,
self.events_ignored_count
);
}
}
@@ -387,7 +388,7 @@ where
count: 0,
min: NTY::zero(),
max: NTY::zero(),
sum: 0f32,
sum: 0.,
sumc: 0,
int_ts,
last_ts: 0,
@@ -402,13 +403,13 @@ where
fn apply_min_max(&mut self, val: NTY) {
if self.count == 0 {
self.min = val.clone();
self.max = val;
self.max = val.clone();
} else {
if val < self.min {
if self.min > val {
self.min = val.clone();
}
if val > self.max {
self.max = val;
if self.max < val {
self.max = val.clone();
}
}
}
@@ -469,6 +470,12 @@ where
let ts = item.tss[i1];
let val = item.values[i1].clone();
if ts < self.int_ts {
if self.last_val.is_none() {
info!(
"ingest_time_weight event before range, only set last ts {} val {:?}",
ts, val
);
}
self.events_ignored_count += 1;
self.last_ts = ts;
self.last_val = Some(val);
@@ -476,8 +483,14 @@ where
self.events_ignored_count += 1;
return;
} else {
debug!("regular");
self.apply_event_time_weight(ts);
if self.last_val.is_none() {
info!(
"call apply_min_max without last val, use current instead {} {:?}",
ts, val
);
self.apply_min_max(val.clone());
}
self.count += 1;
self.last_ts = ts;
self.last_val = Some(val);
@@ -487,24 +500,27 @@ where
}
fn result_reset_unweight(&mut self, range: NanoRange, _expand: bool) -> MinMaxAvgDim0Bins<NTY> {
let avg = if self.sumc == 0 {
0f32
let (min, max, avg) = if self.sumc > 0 {
let avg = self.sum / self.sumc as f32;
(self.min.clone(), self.max.clone(), avg)
} else {
self.sum / self.sumc as f32
let g = match &self.last_val {
Some(x) => x.clone(),
None => NTY::zero(),
};
(g.clone(), g.clone(), g.as_prim_f32())
};
let ret = MinMaxAvgDim0Bins {
ts1s: vec![self.range.beg],
ts2s: vec![self.range.end],
counts: vec![self.count],
mins: vec![self.min.clone()],
maxs: vec![self.max.clone()],
mins: vec![min],
maxs: vec![max],
avgs: vec![avg],
};
self.int_ts = range.beg;
self.range = range;
self.count = 0;
self.min = NTY::zero();
self.max = NTY::zero();
self.sum = 0f32;
self.sumc = 0;
ret
@@ -512,29 +528,33 @@ where
fn result_reset_time_weight(&mut self, range: NanoRange, expand: bool) -> MinMaxAvgDim0Bins<NTY> {
// TODO check callsite for correct expand status.
if true || expand {
if expand {
debug!("result_reset_time_weight calls apply_event_time_weight");
self.apply_event_time_weight(self.range.end);
} else {
debug!("result_reset_time_weight NO EXPAND");
}
let avg = {
let sc = self.range.delta() as f32 * 1e-9;
self.sum / sc
let (min, max, avg) = if self.sumc > 0 {
let avg = self.sum / (self.range.delta() as f32 * 1e-9);
(self.min.clone(), self.max.clone(), avg)
} else {
let g = match &self.last_val {
Some(x) => x.clone(),
None => NTY::zero(),
};
(g.clone(), g.clone(), g.as_prim_f32())
};
let ret = MinMaxAvgDim0Bins {
ts1s: vec![self.range.beg],
ts2s: vec![self.range.end],
counts: vec![self.count],
mins: vec![self.min.clone()],
maxs: vec![self.max.clone()],
mins: vec![min],
maxs: vec![max],
avgs: vec![avg],
};
self.int_ts = range.beg;
self.range = range;
self.count = 0;
self.min = NTY::zero();
self.max = NTY::zero();
self.sum = 0f32;
self.sumc = 0;
ret
@@ -586,8 +606,6 @@ where
impl<NTY: NumOps + 'static> TimeBinnableDyn for ScalarEvents<NTY> {
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Box<dyn TimeBinnerDyn> {
eprintln!("ScalarEvents time_binner_new");
info!("ScalarEvents time_binner_new");
let ret = ScalarEventsTimeBinner::<NTY>::new(edges.into(), do_time_weight);
Box::new(ret)
}
@@ -601,134 +619,74 @@ impl<NTY: NumOps + 'static> EventsDyn for ScalarEvents<NTY> {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnableDyn {
self as &dyn TimeBinnableDyn
}
fn verify(&self) {
let mut ts_max = 0;
for ts in &self.tss {
let ts = *ts;
if ts < ts_max {
error!("unordered event data ts {} ts_max {}", ts, ts_max);
}
ts_max = ts_max.max(ts);
}
}
fn output_info(&self) {
if false {
info!("output_info len {}", self.tss.len());
if self.tss.len() == 1 {
info!(
" only: ts {} pulse {} value {:?}",
self.tss[0], self.pulses[0], self.values[0]
);
} else if self.tss.len() > 1 {
info!(
" first: ts {} pulse {} value {:?}",
self.tss[0], self.pulses[0], self.values[0]
);
let n = self.tss.len() - 1;
info!(
" last: ts {} pulse {} value {:?}",
self.tss[n], self.pulses[n], self.values[n]
);
}
}
}
}
pub struct ScalarEventsTimeBinner<NTY: NumOps> {
// The first two edges are used the next time that we create an aggregator, or push a zero bin.
edges: VecDeque<u64>,
do_time_weight: bool,
range: NanoRange,
agg: Option<EventValuesAggregator<NTY>>,
ready: Option<<EventValuesAggregator<NTY> as TimeBinnableTypeAggregator>::Output>,
}
impl<NTY: NumOps> ScalarEventsTimeBinner<NTY> {
fn new(edges: VecDeque<u64>, do_time_weight: bool) -> Self {
let range = if edges.len() >= 2 {
NanoRange {
beg: edges[0],
end: edges[1],
}
} else {
// Using a dummy for this case.
NanoRange { beg: 1, end: 2 }
};
Self {
edges,
do_time_weight,
range,
agg: None,
ready: None,
}
}
// Move the bin from the current aggregator (if any) to our output collection,
// and step forward in our bin list.
fn cycle(&mut self) {
// TODO expand should be derived from AggKind. Is it still required after all?
let expand = true;
if let Some(agg) = self.agg.as_mut() {
let mut h = agg.result_reset(self.range.clone(), expand);
match self.ready.as_mut() {
Some(fin) => {
fin.append(&mut h);
}
None => {
self.ready = Some(h);
}
}
} else {
let mut h = MinMaxAvgDim0Bins::<NTY>::empty();
h.append_zero(self.range.beg, self.range.end);
match self.ready.as_mut() {
Some(fin) => {
fin.append(&mut h);
}
None => {
self.ready = Some(h);
}
}
}
self.edges.pop_front();
fn next_bin_range(&mut self) -> Option<NanoRange> {
if self.edges.len() >= 2 {
self.range = NanoRange {
let ret = NanoRange {
beg: self.edges[0],
end: self.edges[1],
};
self.edges.pop_front();
Some(ret)
} else {
// Using a dummy for this case.
self.range = NanoRange { beg: 1, end: 2 };
None
}
}
}
impl<NTY: NumOps + 'static> TimeBinnerDyn for ScalarEventsTimeBinner<NTY> {
fn cycle(&mut self) {
Self::cycle(self)
}
fn ingest(&mut self, item: &dyn TimeBinnableDyn) {
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for ScalarEventsTimeBinner no more bin in edges A");
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(self.range.clone()) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for ScalarEventsTimeBinner no more bin in edges B");
return;
}
}
if item.ends_before(self.range.clone()) {
return;
} else {
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for ScalarEventsTimeBinner edge list exhausted");
return;
} else {
if self.agg.is_none() {
self.agg = Some(EventValuesAggregator::new(self.range.clone(), self.do_time_weight));
}
let agg = self.agg.as_mut().unwrap();
if let Some(item) = item
.as_any()
.downcast_ref::<<EventValuesAggregator<NTY> as TimeBinnableTypeAggregator>::Input>()
{
// TODO collect statistics associated with this request:
agg.ingest(item);
} else {
error!("not correct item type");
};
if item.ends_after(self.range.clone()) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for ScalarEventsTimeBinner no more bin in edges C");
return;
}
} else {
break;
}
}
}
}
}
fn bins_ready_count(&self) -> usize {
match &self.ready {
Some(k) => k.len(),
@@ -742,4 +700,133 @@ impl<NTY: NumOps + 'static> TimeBinnerDyn for ScalarEventsTimeBinner<NTY> {
None => None,
}
}
fn ingest(&mut self, item: &dyn TimeBinnableDyn) {
const SELF: &str = "ScalarEventsTimeBinner";
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} no more bin in edges A");
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(NanoRange {
beg: 0,
end: self.edges[1],
}) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} no more bin in edges B");
return;
}
}
if item.ends_before(NanoRange {
beg: self.edges[0],
end: u64::MAX,
}) {
return;
} else {
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} edge list exhausted");
return;
} else {
let agg = if let Some(agg) = self.agg.as_mut() {
agg
} else {
self.agg = Some(EventValuesAggregator::new(
// We know here that we have enough edges for another bin.
// and `next_bin_range` will pop the first edge.
self.next_bin_range().unwrap(),
self.do_time_weight,
));
self.agg.as_mut().unwrap()
};
if let Some(item) = item
.as_any()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_ref::<<EventValuesAggregator<NTY> as TimeBinnableTypeAggregator>::Input>()
{
// TODO collect statistics associated with this request:
agg.ingest(item);
} else {
error!("not correct item type");
};
if item.ends_after(agg.range().clone()) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {SELF} no more bin in edges C");
return;
}
} else {
break;
}
}
}
}
}
fn push_in_progress(&mut self, push_empty: bool) {
// TODO expand should be derived from AggKind. Is it still required after all?
// TODO here, the expand means that agg will assume that the current value is kept constant during
// the rest of the time range.
let expand = true;
let range_next = if self.agg.is_some() {
if let Some(x) = self.next_bin_range() {
Some(x)
} else {
None
}
} else {
None
};
if let Some(agg) = self.agg.as_mut() {
let mut bins;
if let Some(range_next) = range_next {
bins = agg.result_reset(range_next, expand);
} else {
let range_next = NanoRange { beg: 4, end: 5 };
bins = agg.result_reset(range_next, expand);
self.agg = None;
}
assert_eq!(bins.len(), 1);
if push_empty || bins.counts[0] != 0 {
match self.ready.as_mut() {
Some(ready) => {
ready.append(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
}
}
}
fn cycle(&mut self) {
let n = self.bins_ready_count();
self.push_in_progress(true);
if self.bins_ready_count() == n {
if let Some(range) = self.next_bin_range() {
let mut bins = MinMaxAvgDim0Bins::<NTY>::empty();
bins.append_zero(range.beg, range.end);
match self.ready.as_mut() {
Some(ready) => {
ready.append(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
if self.bins_ready_count() <= n {
error!("failed to push a zero bin");
}
} else {
warn!("cycle: no in-progress bin pushed, but also no more bin to add as zero-bin");
}
}
}
}

8
items/src/waveevents.rs
View File

@@ -531,4 +531,12 @@ impl<NTY: NumOps> EventsDyn for WaveEvents<NTY> {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnableDyn {
self as &dyn TimeBinnableDyn
}
fn verify(&self) {
todo!()
}
fn output_info(&self) {
todo!()
}
}