daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor out parts of event bin

Browse Source
This commit is contained in:
Dominik Werder
2023-05-04 15:17:38 +02:00
parent 8fc73fd6ed
commit 585ecfe6a3
9 changed files with 260 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
items_0/src/items_0.rs
View File

@@ -44,6 +44,10 @@ pub trait AppendEmptyBin {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64);
}
pub trait AppendAllFrom {
fn append_all_from(&mut self, src: &mut Self);
}
pub trait AsAnyRef {
fn as_any_ref(&self) -> &dyn Any;
}

5
items_0/src/overlap.rs
View File

@@ -9,11 +9,6 @@ pub trait HasTimestampDeque {
fn pulse_max(&self) -> Option<u64>;
}
pub trait RangeOverlapCmp {
fn range_overlap_cmp_beg(a: u64, b: u64) -> bool;
fn range_overlap_cmp_end(a: u64, b: u64) -> bool;
}
pub trait RangeOverlapInfo {
fn ends_before(&self, range: &SeriesRange) -> bool;
fn ends_after(&self, range: &SeriesRange) -> bool;

32
items_2/src/binsdim0.rs
View File

@@ -18,6 +18,7 @@ use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
use items_0::timebin::TimeBinner;
use items_0::timebin::TimeBins;
use items_0::AppendAllFrom;
use items_0::AppendEmptyBin;
use items_0::AsAnyMut;
use items_0::AsAnyRef;
@@ -106,24 +107,6 @@ impl<NTY: ScalarOps> BinsDim0<NTY> {
self.avgs.push_back(avg);
}
pub fn append_zero(&mut self, beg: u64, end: u64) {
self.ts1s.push_back(beg);
self.ts2s.push_back(end);
self.counts.push_back(0);
self.mins.push_back(NTY::zero_b());
self.maxs.push_back(NTY::zero_b());
self.avgs.push_back(0.);
}
pub fn append_all_from(&mut self, src: &mut Self) {
self.ts1s.extend(src.ts1s.drain(..));
self.ts2s.extend(src.ts2s.drain(..));
self.counts.extend(src.counts.drain(..));
self.mins.extend(src.mins.drain(..));
self.maxs.extend(src.maxs.drain(..));
self.avgs.extend(src.avgs.drain(..));
}
pub fn equal_slack(&self, other: &Self) -> bool {
if self.len() != other.len() {
return false;
@@ -226,6 +209,17 @@ impl<NTY: ScalarOps> AppendEmptyBin for BinsDim0<NTY> {
}
}
impl<NTY: ScalarOps> AppendAllFrom for BinsDim0<NTY> {
fn append_all_from(&mut self, src: &mut Self) {
self.ts1s.extend(src.ts1s.drain(..));
self.ts2s.extend(src.ts2s.drain(..));
self.counts.extend(src.counts.drain(..));
self.mins.extend(src.mins.drain(..));
self.maxs.extend(src.maxs.drain(..));
self.avgs.extend(src.avgs.drain(..));
}
}
impl<NTY: ScalarOps> TimeBins for BinsDim0<NTY> {
fn ts_min(&self) -> Option<u64> {
self.ts1s.front().map(Clone::clone)
@@ -559,7 +553,7 @@ impl<NTY: ScalarOps> TimeBinnableTypeAggregator for BinsDim0Aggregator<NTY> {
todo!()
}
fn result_reset(&mut self, range: SeriesRange, _expand: bool) -> Self::Output {
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
/*if self.sumc > 0 {
self.avg = self.sum / self.sumc as f32;
}

2
items_2/src/binsxbindim0.rs
View File

@@ -537,7 +537,7 @@ impl<NTY: ScalarOps> TimeBinnableTypeAggregator for BinsXbinDim0Aggregator<NTY>
todo!()
}
fn result_reset(&mut self, range: SeriesRange, _expand: bool) -> Self::Output {
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
/*if self.sumc > 0 {
self.avg = self.sum / self.sumc as f32;
}

58
items_2/src/eventsdim0.rs
View File

@@ -1,6 +1,7 @@
use crate::binsdim0::BinsDim0;
use crate::framable::FrameType;
use crate::framable::FrameTypeStatic;
use crate::timebin::TimeBinnerCommonV0Trait;
use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType;
@@ -15,11 +16,12 @@ use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate;
use items_0::framable::FrameTypeInnerStatic;
use items_0::overlap::HasTimestampDeque;
use items_0::overlap::RangeOverlapCmp;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
use items_0::timebin::TimeBinner;
use items_0::AppendAllFrom;
use items_0::AppendEmptyBin;
use items_0::Appendable;
use items_0::AsAnyMut;
use items_0::AsAnyRef;
@@ -713,7 +715,7 @@ impl<STY: ScalarOps> TimeBinnableTypeAggregator for EventsDim0Aggregator<STY> {
}
}
fn result_reset(&mut self, range: SeriesRange, _expand: bool) -> Self::Output {
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
trace!("result_reset {:?}", range);
if self.do_time_weight {
self.result_reset_time_weight(range)
@@ -959,6 +961,50 @@ impl<STY: ScalarOps> EventsDim0TimeBinner<STY> {
}
}
impl<STY: ScalarOps> TimeBinnerCommonV0Trait for EventsDim0TimeBinner<STY> {
type Input = <EventsDim0Aggregator<STY> as TimeBinnableTypeAggregator>::Input;
type Output = <EventsDim0Aggregator<STY> as TimeBinnableTypeAggregator>::Output;
fn type_name() -> &'static str {
Self::type_name()
}
fn common_range_current(&self) -> &SeriesRange {
self.agg.range()
}
fn common_cycle(&mut self) {
todo!()
}
fn common_has_more_range(&self) -> bool {
todo!()
}
fn common_bins_ready_count(&self) -> usize {
todo!()
}
fn common_next_bin_range(&mut self) -> Option<SeriesRange> {
todo!()
}
fn common_take_or_append_all_from(&mut self, mut item: Self::Output) {
match self.ready.as_mut() {
Some(ready) => {
ready.append_all_from(&mut item);
}
None => {
self.ready = Some(item);
}
}
}
fn common_result_reset(&mut self, range: SeriesRange) -> Self::Output {
todo!()
}
}
impl<STY: ScalarOps> TimeBinner for EventsDim0TimeBinner<STY> {
fn bins_ready_count(&self) -> usize {
match &self.ready {
@@ -1042,18 +1088,18 @@ impl<STY: ScalarOps> TimeBinner for EventsDim0TimeBinner<STY> {
// the rest of the time range.
if self.rng.is_none() {
} else {
let expand = true;
let range_next = self.next_bin_range();
self.rng = range_next.clone();
let mut bins = if let Some(range_next) = range_next {
self.agg.result_reset(range_next, expand)
self.agg.result_reset(range_next)
} else {
// Acts as placeholder
// TODO clean up
let range_next = NanoRange {
beg: u64::MAX - 1,
end: u64::MAX,
};
self.agg.result_reset(range_next.into(), expand)
self.agg.result_reset(range_next.into())
};
if bins.len() != 1 {
error!("{self_name}::push_in_progress bins.len() {}", bins.len());
@@ -1085,7 +1131,7 @@ impl<STY: ScalarOps> TimeBinner for EventsDim0TimeBinner<STY> {
if let Some(range) = range_next {
let mut bins = BinsDim0::empty();
if range.is_time() {
bins.append_zero(range.beg_u64(), range.end_u64());
bins.append_empty_bin(range.beg_u64(), range.end_u64());
} else {
error!("TODO {self_name}::cycle is_pulse");
}

3
items_2/src/eventsdim1.rs
View File

@@ -14,7 +14,6 @@ use items_0::collect_s::ToJsonBytes;
use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate;
use items_0::overlap::HasTimestampDeque;
use items_0::overlap::RangeOverlapCmp;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
@@ -638,7 +637,7 @@ impl<NTY: ScalarOps> TimeBinnableTypeAggregator for EventsDim1Aggregator<NTY> {
}
}
fn result_reset(&mut self, range: SeriesRange, expand: bool) -> Self::Output {
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
/*trace!("result_reset {} {}", range.beg, range.end);
if self.do_time_weight {
self.result_reset_time_weight(range, expand)

7
items_2/src/eventsxbindim0.rs
View File

@@ -12,7 +12,6 @@ use items_0::collect_s::ToJsonBytes;
use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate;
use items_0::overlap::HasTimestampDeque;
use items_0::overlap::RangeOverlapCmp;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
@@ -508,14 +507,14 @@ impl<STY: ScalarOps> TimeBinner for EventsXbinDim0TimeBinner<STY> {
trace!("\n+++++\n+++++\n{} range_next {:?}", Self::type_name(), range_next);
self.rng = range_next.clone();
let mut bins = if let Some(range_next) = range_next {
self.agg.result_reset(range_next, expand)
self.agg.result_reset(range_next)
} else {
// Acts as placeholder
let range_next = NanoRange {
beg: u64::MAX - 1,
end: u64::MAX,
};
self.agg.result_reset(range_next.into(), expand)
self.agg.result_reset(range_next.into())
};
if bins.len() != 1 {
error!("{}::push_in_progress bins.len() {}", Self::type_name(), bins.len());
@@ -855,7 +854,7 @@ where
}
}
fn result_reset(&mut self, range: SeriesRange, expand: bool) -> Self::Output {
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
if self.do_time_weight {
self.result_reset_time_weight(range)
} else {

2
items_2/src/items_2.rs
View File

@@ -197,7 +197,7 @@ pub trait TimeBinnableTypeAggregator: Send {
type Output: TimeBinnableType;
fn range(&self) -> &SeriesRange;
fn ingest(&mut self, item: &Self::Input);
fn result_reset(&mut self, range: SeriesRange, expand: bool) -> Self::Output;
fn result_reset(&mut self, range: SeriesRange) -> Self::Output;
}
pub trait ChannelEventsInput: Stream<Item = Sitemty<ChannelEvents>> + EventTransform + Send {}

185
items_2/src/timebin.rs
View File

@@ -1 +1,186 @@
use crate::eventsdim0::EventsDim0TimeBinner;
use items_0::overlap::RangeOverlapInfo;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinnable;
use items_0::AppendEmptyBin;
use items_0::Appendable;
use items_0::Empty;
use items_0::WithLen;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange;
use std::any;
#[allow(unused)]
macro_rules! trace_ingest {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
#[allow(unused)]
macro_rules! trace_ingest_item {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
#[allow(unused)]
macro_rules! trace2 {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
pub trait TimeBinnerCommonV0Trait {
type Input: RangeOverlapInfo + 'static;
type Output: WithLen + Empty + AppendEmptyBin + 'static;
fn type_name() -> &'static str;
fn common_bins_ready_count(&self) -> usize;
fn common_range_current(&self) -> &SeriesRange;
fn common_next_bin_range(&mut self) -> Option<SeriesRange>;
fn common_cycle(&mut self);
fn common_has_more_range(&self) -> bool;
fn common_take_or_append_all_from(&mut self, item: Self::Output);
fn common_result_reset(&mut self, range: SeriesRange) -> Self::Output;
}
pub struct TimeBinnerCommonV0Func {}
impl TimeBinnerCommonV0Func {
pub fn agg_ingest<B>(binner: &mut B, item: &mut <B as TimeBinnerCommonV0Trait>::Input)
where
B: TimeBinnerCommonV0Trait,
{
//self.agg.ingest(item);
todo!()
}
pub fn ingest<B>(binner: &mut B, item: &mut dyn TimeBinnable)
where
B: TimeBinnerCommonV0Trait,
{
let self_name = B::type_name();
trace_ingest_item!(
"TimeBinner for {} ingest agg.range {:?} item {:?}",
Self::type_name(),
self.agg.range(),
item
);
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
// Or consume the input data.
loop {
while item.starts_after(B::common_range_current(binner)) {
trace_ingest_item!("{self_name} ignore item and cycle starts_after");
TimeBinnerCommonV0Func::cycle(binner);
if !B::common_has_more_range(binner) {
debug!("{self_name} no more bin in edges after starts_after");
return;
}
}
if item.ends_before(B::common_range_current(binner)) {
trace_ingest_item!("{self_name} ignore item ends_before");
return;
} else {
if !B::common_has_more_range(binner) {
trace_ingest_item!("{self_name} no more bin in edges");
return;
} else {
if let Some(item) = item
.as_any_mut()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_mut::<B::Input>()
{
// TODO collect statistics associated with this request:
trace_ingest_item!("{self_name} FEED THE ITEM...");
TimeBinnerCommonV0Func::agg_ingest(binner, item);
if item.ends_after(B::common_range_current(binner)) {
trace_ingest_item!(
"{self_name} FED ITEM, ENDS AFTER agg-range {:?}",
B::common_range_current(binner)
);
TimeBinnerCommonV0Func::cycle(binner);
if !B::common_has_more_range(binner) {
warn!("{self_name} no more bin in edges after ingest and cycle");
return;
} else {
trace_ingest_item!("{self_name} item fed, cycled, continue");
}
} else {
trace_ingest_item!("{self_name} item fed, break");
break;
}
} else {
error!("{self_name}::ingest unexpected item type");
};
}
}
}
}
fn push_in_progress<B>(binner: &mut B, push_empty: bool)
where
B: TimeBinnerCommonV0Trait,
{
let self_name = B::type_name();
trace_ingest_item!("{self_name}::push_in_progress push_empty {push_empty}");
// TODO expand should be derived from AggKind. Is it still required after all?
// TODO here, the expand means that agg will assume that the current value is kept constant during
// the rest of the time range.
if B::common_has_more_range(binner) {
let range_next = TimeBinnerCommonV0Trait::common_next_bin_range(binner);
let bins = if let Some(range_next) = range_next {
TimeBinnerCommonV0Trait::common_result_reset(binner, range_next)
//self.agg.result_reset(range_next, expand)
} else {
// Acts as placeholder
// TODO clean up
let range_next = NanoRange {
beg: u64::MAX - 1,
end: u64::MAX,
};
TimeBinnerCommonV0Trait::common_result_reset(binner, range_next.into())
//self.agg.result_reset(range_next.into(), expand)
};
if bins.len() != 1 {
error!("{self_name}::push_in_progress bins.len() {}", bins.len());
return;
} else {
//if push_empty || bins.counts[0] != 0 {
if push_empty {
TimeBinnerCommonV0Trait::common_take_or_append_all_from(binner, bins);
}
}
}
}
fn cycle<B>(binner: &mut B)
where
B: TimeBinnerCommonV0Trait,
{
let self_name = any::type_name::<Self>();
trace_ingest_item!("{self_name}::cycle");
// TODO refactor this logic.
let n = TimeBinnerCommonV0Trait::common_bins_ready_count(binner);
TimeBinnerCommonV0Func::push_in_progress(binner, true);
if TimeBinnerCommonV0Trait::common_bins_ready_count(binner) == n {
let range_next = TimeBinnerCommonV0Trait::common_next_bin_range(binner);
if let Some(range) = range_next {
let mut bins = <B as TimeBinnerCommonV0Trait>::Output::empty();
if range.is_time() {
bins.append_empty_bin(range.beg_u64(), range.end_u64());
} else {
error!("TODO {self_name}::cycle is_pulse");
}
TimeBinnerCommonV0Trait::common_take_or_append_all_from(binner, bins);
if TimeBinnerCommonV0Trait::common_bins_ready_count(binner) <= n {
error!("failed to push a zero bin");
}
} else {
warn!("cycle: no in-progress bin pushed, but also no more bin to add as zero-bin");
}
}
}
}