daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unused

Browse Source
This commit is contained in:
Dominik Werder
2024-10-28 11:04:55 +01:00
parent 52ae20ab02
commit cf3c705823
15 changed files with 102 additions and 1053 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/httpret/src/api4/accounting.rs
-3
View File
@@ -5,7 +5,6 @@ use crate::ReqCtx;
use crate::ServiceSharedResources; use crate::ServiceSharedResources;
use dbconn::worker::PgQueue; use dbconn::worker::PgQueue;
use err::ToPublicError; use err::ToPublicError;
use futures_util::StreamExt;
use http::Method; use http::Method;
use http::StatusCode; use http::StatusCode;
use httpclient::body_empty; use httpclient::body_empty;
@@ -14,8 +13,6 @@ use httpclient::IntoBody;
use httpclient::Requ; use httpclient::Requ;
use httpclient::StreamResponse; use httpclient::StreamResponse;
use httpclient::ToJsonBody; use httpclient::ToJsonBody;
use items_0::Empty;
use items_0::Extendable;
use items_2::accounting::AccountingEvents; use items_2::accounting::AccountingEvents;
use netpod::log::*; use netpod::log::*;
use netpod::req_uri_to_url; use netpod::req_uri_to_url;
crates/httpret/src/api4/binned.rs
+13 -30
View File
@@ -207,22 +207,13 @@ async fn binned_json_single(
span1.in_scope(|| { span1.in_scope(|| {
debug!("begin"); debug!("begin");
}); });
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()); let open_bytes = Arc::pin(OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()));
let open_bytes = Arc::pin(open_bytes);
let open_bytes2 = Arc::pin(OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()));
let (events_read_provider, cache_read_provider) = let (events_read_provider, cache_read_provider) =
make_read_provider(ch_conf.name(), scyqueue, open_bytes2, ctx, ncc); make_read_provider(ch_conf.name(), scyqueue, open_bytes, ctx, ncc);
let item = streams::timebinnedjson::timebinned_json( let item = streams::timebinnedjson::timebinned_json(query, ch_conf, ctx, cache_read_provider, events_read_provider)
query, .instrument(span1)
ch_conf, .await
ctx, .map_err(|e| Error::BinnedStream(e))?;
open_bytes,
cache_read_provider,
events_read_provider,
)
.instrument(span1)
.await
.map_err(|e| Error::BinnedStream(e))?;
match item { match item {
CollectResult::Some(item) => { CollectResult::Some(item) => {
let ret = response(StatusCode::OK) let ret = response(StatusCode::OK)
@@ -272,22 +263,14 @@ async fn binned_json_framed(
span1.in_scope(|| { span1.in_scope(|| {
debug!("begin"); debug!("begin");
}); });
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()); let open_bytes = Arc::pin(OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()));
let open_bytes = Arc::pin(open_bytes);
let open_bytes2 = Arc::pin(OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()));
let (events_read_provider, cache_read_provider) = let (events_read_provider, cache_read_provider) =
make_read_provider(ch_conf.name(), scyqueue, open_bytes2, ctx, ncc); make_read_provider(ch_conf.name(), scyqueue, open_bytes, ctx, ncc);
let stream = streams::timebinnedjson::timebinned_json_framed( let stream =
query, streams::timebinnedjson::timebinned_json_framed(query, ch_conf, ctx, cache_read_provider, events_read_provider)
ch_conf, .instrument(span1)
ctx, .await
open_bytes, .map_err(|e| Error::BinnedStream(e))?;
cache_read_provider,
events_read_provider,
)
.instrument(span1)
.await
.map_err(|e| Error::BinnedStream(e))?;
let stream = bytes_chunks_to_len_framed_str(stream); let stream = bytes_chunks_to_len_framed_str(stream);
let ret = response(StatusCode::OK) let ret = response(StatusCode::OK)
.header(CONTENT_TYPE, APP_JSON_FRAMED) .header(CONTENT_TYPE, APP_JSON_FRAMED)
crates/items_0/src/overlap.rs
-143
View File
@@ -1,5 +1,3 @@
use netpod::range::evrange::SeriesRange;
// TODO rename, no more deque involved // TODO rename, no more deque involved
pub trait HasTimestampDeque { pub trait HasTimestampDeque {
fn timestamp_min(&self) -> Option<u64>; fn timestamp_min(&self) -> Option<u64>;
@@ -7,144 +5,3 @@ pub trait HasTimestampDeque {
fn pulse_min(&self) -> Option<u64>; fn pulse_min(&self) -> Option<u64>;
fn pulse_max(&self) -> Option<u64>; fn pulse_max(&self) -> Option<u64>;
} }
pub trait RangeOverlapInfo {
fn ends_before(&self, range: &SeriesRange) -> bool;
fn ends_after(&self, range: &SeriesRange) -> bool;
fn starts_after(&self, range: &SeriesRange) -> bool;
}
#[macro_export]
macro_rules! impl_range_overlap_info_events {
($ty:ident) => {
impl<STY> RangeOverlapInfo for $ty<STY>
where
STY: ScalarOps,
{
fn ends_before(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max < range.beg_u64()
} else {
true
}
} else if range.is_pulse() {
if let Some(max) = HasTimestampDeque::pulse_max(self) {
max < range.beg_u64()
} else {
true
}
} else {
error!("unexpected");
true
}
}
fn ends_after(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max >= range.end_u64()
} else {
true
}
} else if range.is_pulse() {
if let Some(max) = HasTimestampDeque::pulse_max(self) {
max >= range.end_u64()
} else {
true
}
} else {
error!("unexpected");
false
}
}
fn starts_after(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(min) = HasTimestampDeque::timestamp_min(self) {
min >= range.end_u64()
} else {
true
}
} else if range.is_pulse() {
if let Some(min) = HasTimestampDeque::pulse_min(self) {
min >= range.end_u64()
} else {
true
}
} else {
error!("unexpected");
true
}
}
}
};
}
#[macro_export]
macro_rules! impl_range_overlap_info_bins {
($ty:ident) => {
impl<STY> RangeOverlapInfo for $ty<STY>
where
STY: ScalarOps,
{
fn ends_before(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max <= range.beg_u64()
} else {
true
}
} else if range.is_pulse() {
// TODO for the time being, the ts represent either ts or pulse
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max <= range.beg_u64()
} else {
true
}
} else {
error!("unexpected");
true
}
}
fn ends_after(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max > range.end_u64()
} else {
true
}
} else if range.is_pulse() {
if let Some(max) = HasTimestampDeque::timestamp_max(self) {
max > range.end_u64()
} else {
true
}
} else {
error!("unexpected");
false
}
}
fn starts_after(&self, range: &SeriesRange) -> bool {
if range.is_time() {
if let Some(min) = HasTimestampDeque::timestamp_min(self) {
min >= range.end_u64()
} else {
true
}
} else if range.is_pulse() {
if let Some(min) = HasTimestampDeque::timestamp_min(self) {
min >= range.end_u64()
} else {
true
}
} else {
error!("unexpected");
true
}
}
}
};
}
crates/items_0/src/timebin.rs
+3 -59
View File
@@ -1,28 +1,17 @@
pub mod timebinimpl;
use crate::collect_s::CollectableDyn; use crate::collect_s::CollectableDyn;
use crate::collect_s::CollectorDyn; use crate::collect_s::CollectorDyn;
use crate::collect_s::ToJsonResult; use crate::collect_s::ToJsonResult;
use crate::container::ByteEstimate;
use crate::overlap::RangeOverlapInfo;
use crate::vecpreview::PreviewRange;
use crate::AsAnyMut; use crate::AsAnyMut;
use crate::AsAnyRef; use crate::AsAnyRef;
use crate::Empty;
use crate::Events; use crate::Events;
use crate::Resettable; use crate::Resettable;
use crate::TypeName; use crate::TypeName;
use crate::WithLen; use crate::WithLen;
use err::thiserror;
use err::Error; use err::Error;
use err::ThisError;
use netpod::log::*; use netpod::log::*;
use netpod::range::evrange::SeriesRange;
use netpod::BinnedRange; use netpod::BinnedRange;
use netpod::BinnedRangeEnum; use netpod::BinnedRangeEnum;
use netpod::TsNano; use netpod::TsNano;
use serde::Deserialize;
use serde::Serialize;
use std::any::Any; use std::any::Any;
use std::fmt; use std::fmt;
use std::ops::Range; use std::ops::Range;
@@ -170,20 +159,6 @@ impl Clone for Box<dyn TimeBinned> {
} }
} }
impl RangeOverlapInfo for Box<dyn TimeBinned> {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
}
}
impl TimeBinnable for Box<dyn TimeBinned> { impl TimeBinnable for Box<dyn TimeBinned> {
fn time_binner_new( fn time_binner_new(
&self, &self,
@@ -230,9 +205,7 @@ pub trait TimeBinner: fmt::Debug + Send {
/// Provides a time-binned representation of the implementing type. /// Provides a time-binned representation of the implementing type.
/// In contrast to `TimeBinnableType` this is meant for trait objects. /// In contrast to `TimeBinnableType` this is meant for trait objects.
pub trait TimeBinnable: pub trait TimeBinnable: fmt::Debug + WithLen + CollectableDyn + Any + AsAnyRef + AsAnyMut + Send {
fmt::Debug + WithLen + RangeOverlapInfo + CollectableDyn + Any + AsAnyRef + AsAnyMut + Send
{
// TODO implementors may fail if edges contain not at least 2 entries. // TODO implementors may fail if edges contain not at least 2 entries.
fn time_binner_new( fn time_binner_new(
&self, &self,
@@ -252,21 +225,6 @@ impl WithLen for Box<dyn TimeBinnable> {
} }
} }
#[allow(unused)]
impl RangeOverlapInfo for Box<dyn TimeBinnable> {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
}
}
impl TimeBinnable for Box<dyn TimeBinnable> { impl TimeBinnable for Box<dyn TimeBinnable> {
fn time_binner_new( fn time_binner_new(
&self, &self,
@@ -286,20 +244,6 @@ impl TimeBinnable for Box<dyn TimeBinnable> {
} }
} }
impl RangeOverlapInfo for Box<dyn Events> {
fn ends_before(&self, range: &SeriesRange) -> bool {
RangeOverlapInfo::ends_before(self.as_ref(), range)
}
fn ends_after(&self, range: &SeriesRange) -> bool {
RangeOverlapInfo::ends_after(self.as_ref(), range)
}
fn starts_after(&self, range: &SeriesRange) -> bool {
RangeOverlapInfo::starts_after(self.as_ref(), range)
}
}
impl TimeBinnable for Box<dyn Events> { impl TimeBinnable for Box<dyn Events> {
fn time_binner_new( fn time_binner_new(
&self, &self,
@@ -425,7 +369,7 @@ impl TimeBinnerTy for TimeBinnerDynStruct {
} }
impl TimeBinner for TimeBinnerDynStruct { impl TimeBinner for TimeBinnerDynStruct {
fn ingest(&mut self, item: &mut dyn TimeBinnable) { fn ingest(&mut self, _item: &mut dyn TimeBinnable) {
todo!() todo!()
} }
@@ -437,7 +381,7 @@ impl TimeBinner for TimeBinnerDynStruct {
todo!() todo!()
} }
fn push_in_progress(&mut self, push_empty: bool) { fn push_in_progress(&mut self, _push_empty: bool) {
todo!() todo!()
} }
crates/items_0/src/timebin/timebinimpl.rs
-199
View File
@@ -1,199 +0,0 @@
#![allow(unused)]
use crate::timebin::TimeBinnable;
use crate::timebin::TimeBinned;
use crate::timebin::TimeBinner;
use crate::timebin::TimeBinnerIngest;
use crate::TypeName;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
#[allow(unused)]
macro_rules! trace2 {
($($arg:tt)*) => { trace!($($arg)*) };
}
#[allow(unused)]
macro_rules! trace_ingest {
($($arg:tt)*) => { trace!($($arg)*) };
}
#[cfg(target_abi = "x32")]
impl<T> TimeBinner for T
where
T: TimeBinnerIngest,
{
fn bins_ready_count(&self) -> usize {
match &self.ready {
Some(k) => k.len(),
None => 0,
}
}
fn bins_ready(&mut self) -> Option<Box<dyn TimeBinned>> {
match self.ready.take() {
Some(k) => Some(Box::new(k)),
None => None,
}
}
fn ingest(&mut self, item: &mut dyn TimeBinnable) {
trace2!(
"TimeBinner for {} ingest agg.range {:?} item {:?}",
self.type_name(),
self.agg.range(),
item
);
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(self.agg.range()) {
trace!(
"{} IGNORE ITEM AND CYCLE BECAUSE item.starts_after",
self.type_name()
);
self.cycle();
if self.rng.is_none() {
warn!("{} no more bin in edges B", self.type_name());
return;
}
}
if item.ends_before(self.agg.range()) {
trace!("{} IGNORE ITEM BECAUSE ends_before", self.type_name());
return;
} else {
if self.rng.is_none() {
trace!("{} no more bin in edges D", self.type_name());
return;
} else {
match TimeBinnerIngest::ingest_inrange(self, item) {
Ok(()) => {
if item.ends_after(self.agg.range()) {
trace_ingest!("{} FED ITEM, ENDS AFTER.", self.type_name());
self.cycle();
if self.rng.is_none() {
warn!("{} no more bin in edges C", self.type_name());
return;
} else {
trace_ingest!("{} FED ITEM, CYCLED, CONTINUE.", self.type_name());
}
} else {
trace_ingest!("{} FED ITEM.", self.type_name());
break;
}
}
Err(e) => {
error!("{}::ingest {}", self.type_name(), e);
}
}
/*
// Move to TimeBinnerIngest
if let Some(item) = item
.as_any_ref()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_ref::<<EventsDim0Aggregator<STY> as TimeBinnableTypeAggregator>::Input>()
{
// TODO collect statistics associated with this request:
trace_ingest!("{self_name} FEED THE ITEM...");
self.agg.ingest(item);
} else {
error!("{self_name}::ingest unexpected item type");
};
*/
}
}
}
}
fn push_in_progress(&mut self, push_empty: bool) {
trace!("{}::push_in_progress push_empty {push_empty}", self.type_name());
// TODO expand should be derived from AggKind. Is it still required after all?
// TODO here, the expand means that agg will assume that the current value is kept constant during
// the rest of the time range.
if self.rng.is_none() {
} else {
let expand = true;
let range_next = self.next_bin_range();
self.rng = range_next.clone();
let mut bins = if let Some(range_next) = range_next {
self.agg.result_reset(range_next, expand)
} else {
// Acts as placeholder
let range_next = NanoRange {
beg: u64::MAX - 1,
end: u64::MAX,
};
self.agg.result_reset(range_next.into(), expand)
};
if bins.len() != 1 {
error!("{}::push_in_progress bins.len() {}", self.type_name(), bins.len());
return;
} else {
if push_empty || bins.counts[0] != 0 {
match self.ready.as_mut() {
Some(ready) => {
ready.append_all_from(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
}
}
}
}
fn cycle(&mut self) {
trace!("{}::cycle", self.type_name());
// TODO refactor this logic.
let n = self.bins_ready_count();
self.push_in_progress(true);
if self.bins_ready_count() == n {
let range_next = self.next_bin_range();
self.rng = range_next.clone();
if let Some(range) = range_next {
/*
TODO Move out to trait.
let mut bins = BinsDim0::empty();
if range.is_time() {
bins.append_zero(range.beg_u64(), range.end_u64());
} else {
error!("TODO {self_name}::cycle is_pulse");
}
match self.ready.as_mut() {
Some(ready) => {
ready.append_all_from(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
*/
if self.bins_ready_count() <= n {
error!("{}::cycle failed to push a zero bin", self.type_name());
}
} else {
warn!(
"{}::cycle no in-progress bin pushed, but also no more bin to add as zero-bin",
self.type_name()
);
}
}
}
fn set_range_complete(&mut self) {
self.range_final = true;
}
fn empty(&self) -> Box<dyn TimeBinned> {
/*
TODO factor out to trait.
let ret = <EventsDim0Aggregator<STY> as TimeBinnableTypeAggregator>::Output::empty();
*/
let ret = todo!();
Box::new(ret)
}
}
crates/items_2/src/binsdim0.rs
+3 -112
View File
@@ -3,11 +3,8 @@ use crate::timebin::TimeBinnerCommonV0Trait;
use crate::ts_offs_from_abs; use crate::ts_offs_from_abs;
use crate::ts_offs_from_abs_with_anchor; use crate::ts_offs_from_abs_with_anchor;
use crate::IsoDateTime; use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType; use crate::TimeBinnableType;
use crate::TimeBinnableTypeAggregator; use crate::TimeBinnableTypeAggregator;
use chrono::TimeZone;
use chrono::Utc;
use err::Error; use err::Error;
use items_0::collect_s::CollectableDyn; use items_0::collect_s::CollectableDyn;
use items_0::collect_s::CollectableType; use items_0::collect_s::CollectableType;
@@ -285,8 +282,6 @@ impl<STY: ScalarOps> HasTimestampDeque for BinsDim0<STY> {
} }
} }
items_0::impl_range_overlap_info_bins!(BinsDim0);
impl<NTY: ScalarOps> AppendEmptyBin for BinsDim0<NTY> { impl<NTY: ScalarOps> AppendEmptyBin for BinsDim0<NTY> {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64) { fn append_empty_bin(&mut self, ts1: u64, ts2: u64) {
debug!("AppendEmptyBin::append_empty_bin should not get used"); debug!("AppendEmptyBin::append_empty_bin should not get used");
@@ -1230,121 +1225,17 @@ impl<NTY: ScalarOps> TimeBinner for BinsDim0TimeBinner<NTY> {
} }
fn ingest(&mut self, item: &mut dyn TimeBinnable) { fn ingest(&mut self, item: &mut dyn TimeBinnable) {
/*let self_name = any::type_name::<Self>(); panic!("TODO do not use")
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {self_name} no more bin in edges A\n{:?}\n\n", item);
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(NanoRange {
beg: 0,
end: self.edges[1],
}) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {self_name} no more bin in edges B\n{:?}\n\n", item);
return;
}
}
if item.ends_before(NanoRange {
beg: self.edges[0],
end: u64::MAX,
}) {
return;
} else {
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {self_name} edge list exhausted");
return;
} else {
let agg = if let Some(agg) = self.agg.as_mut() {
agg
} else {
self.agg = Some(BinsDim0Aggregator::new(
// We know here that we have enough edges for another bin.
// and `next_bin_range` will pop the first edge.
self.next_bin_range().unwrap(),
self.do_time_weight,
));
self.agg.as_mut().unwrap()
};
if let Some(item) = item
.as_any_ref()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_ref::<<BinsDim0Aggregator<NTY> as TimeBinnableTypeAggregator>::Input>()
{
agg.ingest(item);
} else {
let tyid_item = std::any::Any::type_id(item.as_any_ref());
error!("not correct item type {:?}", tyid_item);
};
if item.ends_after(agg.range().clone()) {
self.cycle();
if self.edges.len() < 2 {
warn!("TimeBinnerDyn for {self_name} no more bin in edges C\n{:?}\n\n", item);
return;
}
} else {
break;
}
}
}
}*/
TimeBinnerCommonV0Func::ingest(self, item)
} }
// TODO there is too much common code between implementors: // TODO there is too much common code between implementors:
fn push_in_progress(&mut self, push_empty: bool) { fn push_in_progress(&mut self, push_empty: bool) {
// TODO expand should be derived from AggKind. Is it still required after all? panic!("TODO do not use")
/*let expand = true;
if let Some(agg) = self.agg.as_mut() {
let dummy_range = NanoRange { beg: 4, end: 5 };
let mut bins = agg.result_reset(dummy_range, expand);
self.agg = None;
assert_eq!(bins.len(), 1);
if push_empty || bins.counts[0] != 0 {
match self.ready.as_mut() {
Some(ready) => {
ready.append_all_from(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
}
}*/
TimeBinnerCommonV0Func::push_in_progress(self, push_empty)
} }
// TODO there is too much common code between implementors: // TODO there is too much common code between implementors:
fn cycle(&mut self) { fn cycle(&mut self) {
/*let n = self.bins_ready_count(); panic!("TODO do not use")
self.push_in_progress(true);
if self.bins_ready_count() == n {
if let Some(range) = self.next_bin_range() {
let mut bins = BinsDim0::<NTY>::empty();
bins.append_zero(range.beg, range.end);
match self.ready.as_mut() {
Some(ready) => {
ready.append_all_from(&mut bins);
}
None => {
self.ready = Some(bins);
}
}
if self.bins_ready_count() <= n {
error!("failed to push a zero bin");
}
} else {
warn!("cycle: no in-progress bin pushed, but also no more bin to add as zero-bin");
}
}*/
TimeBinnerCommonV0Func::cycle(self)
} }
fn set_range_complete(&mut self) { fn set_range_complete(&mut self) {
crates/items_2/src/binsxbindim0.rs
-16
View File
@@ -1,10 +1,8 @@
use crate::ts_offs_from_abs; use crate::ts_offs_from_abs;
use crate::ts_offs_from_abs_with_anchor; use crate::ts_offs_from_abs_with_anchor;
use crate::IsoDateTime; use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType; use crate::TimeBinnableType;
use crate::TimeBinnableTypeAggregator; use crate::TimeBinnableTypeAggregator;
use chrono::{TimeZone, Utc};
use err::Error; use err::Error;
use items_0::collect_s::CollectableDyn; use items_0::collect_s::CollectableDyn;
use items_0::collect_s::CollectableType; use items_0::collect_s::CollectableType;
@@ -235,20 +233,6 @@ impl<STY> Resettable for BinsXbinDim0<STY> {
} }
} }
impl<NTY> RangeOverlapInfo for BinsXbinDim0<NTY> {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
}
}
impl<NTY: ScalarOps> AppendEmptyBin for BinsXbinDim0<NTY> { impl<NTY: ScalarOps> AppendEmptyBin for BinsXbinDim0<NTY> {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64) { fn append_empty_bin(&mut self, ts1: u64, ts2: u64) {
self.ts1s.push_back(ts1); self.ts1s.push_back(ts1);
crates/items_2/src/channelevents.rs
+10 -25
View File
@@ -7,7 +7,6 @@ use items_0::collect_s::CollectorDyn;
use items_0::container::ByteEstimate; use items_0::container::ByteEstimate;
use items_0::framable::FrameTypeInnerStatic; use items_0::framable::FrameTypeInnerStatic;
use items_0::isodate::IsoDateTime; use items_0::isodate::IsoDateTime;
use items_0::overlap::RangeOverlapInfo;
use items_0::streamitem::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID; use items_0::streamitem::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
use items_0::timebin::TimeBinnable; use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinnableTy; use items_0::timebin::TimeBinnableTy;
@@ -827,20 +826,6 @@ impl Mergeable for ChannelEvents {
} }
} }
impl RangeOverlapInfo for ChannelEvents {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
}
}
impl TimeBinnable for ChannelEvents { impl TimeBinnable for ChannelEvents {
fn time_binner_new( fn time_binner_new(
&self, &self,
@@ -865,7 +850,7 @@ impl EventsNonObj for ChannelEvents {
fn into_tss_pulses(self: Box<Self>) -> (VecDeque<u64>, VecDeque<u64>) { fn into_tss_pulses(self: Box<Self>) -> (VecDeque<u64>, VecDeque<u64>) {
match *self { match *self {
ChannelEvents::Events(k) => k.into_tss_pulses(), ChannelEvents::Events(k) => k.into_tss_pulses(),
ChannelEvents::Status(k) => (VecDeque::new(), VecDeque::new()), ChannelEvents::Status(_) => (VecDeque::new(), VecDeque::new()),
} }
} }
} }
@@ -910,7 +895,7 @@ impl Events for ChannelEvents {
todo!() todo!()
} }
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events> { fn take_new_events_until_ts(&mut self, _ts_end: u64) -> Box<dyn Events> {
todo!() todo!()
} }
@@ -937,15 +922,15 @@ impl Events for ChannelEvents {
} }
} }
fn find_lowest_index_gt_evs(&self, ts: u64) -> Option<usize> { fn find_lowest_index_gt_evs(&self, _ts: u64) -> Option<usize> {
todo!() todo!()
} }
fn find_lowest_index_ge_evs(&self, ts: u64) -> Option<usize> { fn find_lowest_index_ge_evs(&self, _ts: u64) -> Option<usize> {
todo!() todo!()
} }
fn find_highest_index_lt_evs(&self, ts: u64) -> Option<usize> { fn find_highest_index_lt_evs(&self, _ts: u64) -> Option<usize> {
todo!() todo!()
} }
@@ -953,7 +938,7 @@ impl Events for ChannelEvents {
todo!() todo!()
} }
fn partial_eq_dyn(&self, other: &dyn Events) -> bool { fn partial_eq_dyn(&self, _other: &dyn Events) -> bool {
todo!() todo!()
} }
@@ -990,7 +975,7 @@ impl Events for ChannelEvents {
fn to_json_string(&self) -> String { fn to_json_string(&self) -> String {
match self { match self {
ChannelEvents::Events(item) => item.to_json_string(), ChannelEvents::Events(item) => item.to_json_string(),
ChannelEvents::Status(item) => { ChannelEvents::Status(_item) => {
error!("TODO convert status to json"); error!("TODO convert status to json");
String::new() String::new()
} }
@@ -1000,7 +985,7 @@ impl Events for ChannelEvents {
fn to_json_vec_u8(&self) -> Vec<u8> { fn to_json_vec_u8(&self) -> Vec<u8> {
match self { match self {
ChannelEvents::Events(item) => item.to_json_vec_u8(), ChannelEvents::Events(item) => item.to_json_vec_u8(),
ChannelEvents::Status(item) => { ChannelEvents::Status(_item) => {
error!("TODO convert status to json"); error!("TODO convert status to json");
Vec::new() Vec::new()
} }
@@ -1010,7 +995,7 @@ impl Events for ChannelEvents {
fn to_cbor_vec_u8(&self) -> Vec<u8> { fn to_cbor_vec_u8(&self) -> Vec<u8> {
match self { match self {
ChannelEvents::Events(item) => item.to_cbor_vec_u8(), ChannelEvents::Events(item) => item.to_cbor_vec_u8(),
ChannelEvents::Status(item) => { ChannelEvents::Status(_item) => {
error!("TODO convert status to cbor"); error!("TODO convert status to cbor");
Vec::new() Vec::new()
} }
@@ -1030,7 +1015,7 @@ impl Events for ChannelEvents {
use ChannelEvents::*; use ChannelEvents::*;
match self { match self {
Events(x) => x.to_dim0_f32_for_binning(), Events(x) => x.to_dim0_f32_for_binning(),
Status(x) => panic!("ChannelEvents::to_dim0_f32_for_binning"), Status(_x) => panic!("ChannelEvents::to_dim0_f32_for_binning"),
} }
} }
crates/items_2/src/eventsdim0.rs
+7 -39
View File
@@ -1,6 +1,4 @@
use crate::binsdim0::BinsDim0; use crate::binsdim0::BinsDim0;
use crate::framable::FrameType;
use crate::framable::FrameTypeStatic;
use crate::timebin::ChooseIndicesForTimeBin; use crate::timebin::ChooseIndicesForTimeBin;
use crate::timebin::ChooseIndicesForTimeBinEvents; use crate::timebin::ChooseIndicesForTimeBinEvents;
use crate::timebin::TimeAggregatorCommonV0Func; use crate::timebin::TimeAggregatorCommonV0Func;
@@ -8,44 +6,35 @@ use crate::timebin::TimeAggregatorCommonV0Trait;
use crate::timebin::TimeBinnerCommonV0Func; use crate::timebin::TimeBinnerCommonV0Func;
use crate::timebin::TimeBinnerCommonV0Trait; use crate::timebin::TimeBinnerCommonV0Trait;
use crate::IsoDateTime; use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType; use crate::TimeBinnableType;
use crate::TimeBinnableTypeAggregator; use crate::TimeBinnableTypeAggregator;
use err::Error; use err::Error;
use items_0::collect_s::CollectableDyn; use items_0::collect_s::CollectableDyn;
use items_0::collect_s::CollectedDyn; use items_0::collect_s::CollectedDyn;
use items_0::collect_s::CollectorDyn;
use items_0::collect_s::CollectorTy; use items_0::collect_s::CollectorTy;
use items_0::collect_s::ToJsonBytes;
use items_0::collect_s::ToJsonResult; use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate; use items_0::container::ByteEstimate;
use items_0::framable::FrameTypeInnerStatic;
use items_0::overlap::HasTimestampDeque; use items_0::overlap::HasTimestampDeque;
use items_0::scalar_ops::ScalarOps; use items_0::scalar_ops::ScalarOps;
use items_0::test::f32_iter_cmp_near;
use items_0::timebin::TimeBinnable; use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned; use items_0::timebin::TimeBinned;
use items_0::timebin::TimeBinner; use items_0::timebin::TimeBinner;
use items_0::AppendAllFrom; use items_0::AppendAllFrom;
use items_0::AppendEmptyBin;
use items_0::Appendable; use items_0::Appendable;
use items_0::AsAnyMut; use items_0::AsAnyMut;
use items_0::AsAnyRef; use items_0::AsAnyRef;
use items_0::Empty; use items_0::Empty;
use items_0::Events; use items_0::Events;
use items_0::EventsNonObj; use items_0::EventsNonObj;
use items_0::HasNonemptyFirstBin;
use items_0::MergeError; use items_0::MergeError;
use items_0::Resettable; use items_0::Resettable;
use items_0::TypeName; use items_0::TypeName;
use items_0::WithLen; use items_0::WithLen;
use netpod::is_false; use netpod::is_false;
use netpod::log::*; use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange; use netpod::range::evrange::SeriesRange;
use netpod::timeunits::MS; use netpod::timeunits::MS;
use netpod::timeunits::SEC; use netpod::timeunits::SEC;
use netpod::BinnedRange;
use netpod::BinnedRangeEnum; use netpod::BinnedRangeEnum;
use netpod::TsNano; use netpod::TsNano;
use serde::Deserialize; use serde::Deserialize;
@@ -244,8 +233,6 @@ impl<STY: ScalarOps> HasTimestampDeque for EventsDim0<STY> {
} }
} }
items_0::impl_range_overlap_info_events!(EventsDim0);
impl<STY> ChooseIndicesForTimeBin for EventsDim0<STY> { impl<STY> ChooseIndicesForTimeBin for EventsDim0<STY> {
fn choose_indices_unweight(&self, beg: u64, end: u64) -> (Option<usize>, usize, usize) { fn choose_indices_unweight(&self, beg: u64, end: u64) -> (Option<usize>, usize, usize) {
ChooseIndicesForTimeBinEvents::choose_unweight(beg, end, &self.tss) ChooseIndicesForTimeBinEvents::choose_unweight(beg, end, &self.tss)
@@ -1431,30 +1418,6 @@ mod test_serde_opt {
} }
} }
#[test]
fn overlap_info_00() {
let mut ev1 = EventsDim0::empty();
ev1.push(MS * 1200, 3, 1.2f32);
ev1.push(MS * 3200, 3, 3.2f32);
let range = SeriesRange::TimeRange(NanoRange {
beg: MS * 1000,
end: MS * 2000,
});
assert_eq!(ev1.ends_after(&range), true);
}
#[test]
fn overlap_info_01() {
let mut ev1 = EventsDim0::empty();
ev1.push(MS * 1200, 3, 1.2f32);
ev1.push(MS * 1400, 3, 3.2f32);
let range = SeriesRange::TimeRange(NanoRange {
beg: MS * 1000,
end: MS * 2000,
});
assert_eq!(ev1.ends_after(&range), false);
}
#[test] #[test]
fn binner_00() { fn binner_00() {
let mut ev1 = EventsDim0::empty(); let mut ev1 = EventsDim0::empty();
@@ -1553,7 +1516,7 @@ fn bin_binned_02() {
#[test] #[test]
fn events_timebin_ingest_continuous_00() { fn events_timebin_ingest_continuous_00() {
let binrange = BinnedRangeEnum::Time(BinnedRange { let binrange = BinnedRangeEnum::Time(netpod::BinnedRange {
bin_len: TsNano::from_ns(SEC * 2), bin_len: TsNano::from_ns(SEC * 2),
bin_off: 9, bin_off: 9,
bin_cnt: 20, bin_cnt: 20,
@@ -1573,5 +1536,10 @@ fn events_timebin_ingest_continuous_00() {
let mut exp = BinsDim0::empty(); let mut exp = BinsDim0::empty();
// exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None); // exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None);
exp.push(SEC * 20, SEC * 22, 1, 20, 20, 20., 20); exp.push(SEC * 20, SEC * 22, 1, 20, 20, 20., 20);
assert!(f32_iter_cmp_near(got.avgs.clone(), exp.avgs.clone(), 0.0001, 0.0001)); assert!(items_0::test::f32_iter_cmp_near(
got.avgs.clone(),
exp.avgs.clone(),
0.0001,
0.0001
));
} }
crates/items_2/src/eventsdim0enum.rs
-15
View File
@@ -7,7 +7,6 @@ use items_0::collect_s::ToJsonBytes;
use items_0::collect_s::ToJsonResult; use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate; use items_0::container::ByteEstimate;
use items_0::isodate::IsoDateTime; use items_0::isodate::IsoDateTime;
use items_0::overlap::RangeOverlapInfo;
use items_0::scalar_ops::ScalarOps; use items_0::scalar_ops::ScalarOps;
use items_0::timebin::TimeBinnable; use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinnableTy; use items_0::timebin::TimeBinnableTy;
@@ -284,20 +283,6 @@ impl EventsNonObj for EventsDim0Enum {
} }
} }
impl RangeOverlapInfo for EventsDim0Enum {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
}
}
// NOTE just a dummy because currently we don't use this for time binning // NOTE just a dummy because currently we don't use this for time binning
#[derive(Debug)] #[derive(Debug)]
pub struct EventsDim0EnumTimeBinner; pub struct EventsDim0EnumTimeBinner;
crates/items_2/src/eventsdim1.rs
+1 -4
View File
@@ -1,7 +1,6 @@
use crate::binsdim0::BinsDim0; use crate::binsdim0::BinsDim0;
use crate::eventsxbindim0::EventsXbinDim0; use crate::eventsxbindim0::EventsXbinDim0;
use crate::IsoDateTime; use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType; use crate::TimeBinnableType;
use crate::TimeBinnableTypeAggregator; use crate::TimeBinnableTypeAggregator;
use err::Error; use err::Error;
@@ -187,8 +186,6 @@ impl<STY: ScalarOps> HasTimestampDeque for EventsDim1<STY> {
} }
} }
items_0::impl_range_overlap_info_events!(EventsDim1);
impl<STY> TimeBinnableType for EventsDim1<STY> impl<STY> TimeBinnableType for EventsDim1<STY>
where where
STY: ScalarOps, STY: ScalarOps,
@@ -196,7 +193,7 @@ where
type Output = BinsDim0<STY>; type Output = BinsDim0<STY>;
type Aggregator = EventsDim1Aggregator<STY>; type Aggregator = EventsDim1Aggregator<STY>;
fn aggregator(range: SeriesRange, x_bin_count: usize, do_time_weight: bool) -> Self::Aggregator { fn aggregator(_range: SeriesRange, _x_bin_count: usize, _do_time_weight: bool) -> Self::Aggregator {
panic!("TODO remove") panic!("TODO remove")
} }
} }
crates/items_2/src/eventsxbindim0.rs
+1 -68
View File
@@ -1,6 +1,5 @@
use crate::binsxbindim0::BinsXbinDim0; use crate::binsxbindim0::BinsXbinDim0;
use crate::IsoDateTime; use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType; use crate::TimeBinnableType;
use crate::TimeBinnableTypeAggregator; use crate::TimeBinnableTypeAggregator;
use err::Error; use err::Error;
@@ -8,7 +7,6 @@ use items_0::collect_s::CollectableDyn;
use items_0::collect_s::CollectableType; use items_0::collect_s::CollectableType;
use items_0::collect_s::CollectedDyn; use items_0::collect_s::CollectedDyn;
use items_0::collect_s::CollectorTy; use items_0::collect_s::CollectorTy;
use items_0::collect_s::ToJsonBytes;
use items_0::collect_s::ToJsonResult; use items_0::collect_s::ToJsonResult;
use items_0::container::ByteEstimate; use items_0::container::ByteEstimate;
use items_0::overlap::HasTimestampDeque; use items_0::overlap::HasTimestampDeque;
@@ -187,8 +185,6 @@ impl<STY: ScalarOps> HasTimestampDeque for EventsXbinDim0<STY> {
} }
} }
items_0::impl_range_overlap_info_events!(EventsXbinDim0);
impl<STY: ScalarOps> EventsNonObj for EventsXbinDim0<STY> { impl<STY: ScalarOps> EventsNonObj for EventsXbinDim0<STY> {
fn into_tss_pulses(self: Box<Self>) -> (VecDeque<u64>, VecDeque<u64>) { fn into_tss_pulses(self: Box<Self>) -> (VecDeque<u64>, VecDeque<u64>) {
info!( info!(
@@ -455,70 +451,7 @@ impl<STY: ScalarOps> TimeBinner for EventsXbinDim0TimeBinner<STY> {
} }
fn ingest(&mut self, item: &mut dyn TimeBinnable) { fn ingest(&mut self, item: &mut dyn TimeBinnable) {
trace2!( panic!("TODO remove TimeBinner for EventsXbinDim0TimeBinner ingest")
"TimeBinner for {} ingest agg.range {:?} item {:?}",
Self::type_name(),
self.agg.range(),
item
);
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
loop {
while item.starts_after(self.agg.range()) {
trace!(
"{} IGNORE ITEM AND CYCLE BECAUSE item.starts_after",
Self::type_name()
);
self.cycle();
if self.rng.is_none() {
warn!("{} no more bin in edges B", Self::type_name());
return;
}
}
if item.ends_before(self.agg.range()) {
trace!(
"{} IGNORE ITEM BECAUSE ends_before {:?} {:?}",
Self::type_name(),
self.agg.range(),
item
);
return;
} else {
if self.rng.is_none() {
trace!("{} no more bin in edges D", Self::type_name());
return;
} else {
if let Some(item) = item
.as_any_ref()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_ref::<<EventsXbinDim0Aggregator<STY> as TimeBinnableTypeAggregator>::Input>()
{
// TODO collect statistics associated with this request:
trace_ingest!("{} FEED THE ITEM...", Self::type_name());
self.agg.ingest(item);
if item.ends_after(self.agg.range()) {
trace_ingest!("{} FED ITEM, ENDS AFTER.", Self::type_name());
self.cycle();
if self.rng.is_none() {
warn!("{} no more bin in edges C", Self::type_name());
return;
} else {
trace_ingest!("{} FED ITEM, CYCLED, CONTINUE.", Self::type_name());
}
} else {
trace_ingest!("{} FED ITEM.", Self::type_name());
break;
}
} else {
error!("{}::ingest unexpected item type", Self::type_name());
};
}
}
}
} }
fn push_in_progress(&mut self, push_empty: bool) { fn push_in_progress(&mut self, push_empty: bool) {
crates/items_2/src/items_2.rs
+1 -2
View File
@@ -24,7 +24,6 @@ pub mod transform;
use channelevents::ChannelEvents; use channelevents::ChannelEvents;
use futures_util::Stream; use futures_util::Stream;
use items_0::isodate::IsoDateTime; use items_0::isodate::IsoDateTime;
use items_0::overlap::RangeOverlapInfo;
use items_0::streamitem::Sitemty; use items_0::streamitem::Sitemty;
use items_0::transform::EventTransform; use items_0::transform::EventTransform;
use items_0::Empty; use items_0::Empty;
@@ -175,7 +174,7 @@ impl Mergeable for Box<dyn Events> {
} }
// TODO rename to `Typed` // TODO rename to `Typed`
pub trait TimeBinnableType: Send + Unpin + RangeOverlapInfo + Empty { pub trait TimeBinnableType: Send + Unpin + Empty {
type Output: TimeBinnableType; type Output: TimeBinnableType;
type Aggregator: TimeBinnableTypeAggregator<Input = Self, Output = Self::Output> + Send + Unpin; type Aggregator: TimeBinnableTypeAggregator<Input = Self, Output = Self::Output> + Send + Unpin;
fn aggregator(range: SeriesRange, bin_count: usize, do_time_weight: bool) -> Self::Aggregator; fn aggregator(range: SeriesRange, bin_count: usize, do_time_weight: bool) -> Self::Aggregator;
crates/items_2/src/timebin.rs
+3 -86
View File
@@ -1,4 +1,3 @@
use items_0::overlap::RangeOverlapInfo;
use items_0::timebin::TimeBinnable; use items_0::timebin::TimeBinnable;
use items_0::AppendEmptyBin; use items_0::AppendEmptyBin;
use items_0::Empty; use items_0::Empty;
@@ -23,7 +22,7 @@ macro_rules! trace_ingest_event { ($($arg:tt)*) => ( if false { trace!($($arg)*)
macro_rules! trace_ingest_detail { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) } macro_rules! trace_ingest_detail { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
pub trait TimeBinnerCommonV0Trait { pub trait TimeBinnerCommonV0Trait {
type Input: RangeOverlapInfo + 'static; type Input: 'static;
type Output: WithLen + Empty + AppendEmptyBin + HasNonemptyFirstBin + 'static; type Output: WithLen + Empty + AppendEmptyBin + HasNonemptyFirstBin + 'static;
fn type_name() -> &'static str; fn type_name() -> &'static str;
fn common_bins_ready_count(&self) -> usize; fn common_bins_ready_count(&self) -> usize;
@@ -45,89 +44,7 @@ impl TimeBinnerCommonV0Func {
where where
B: TimeBinnerCommonV0Trait, B: TimeBinnerCommonV0Trait,
{ {
let self_name = B::type_name(); panic!("TimeBinnerCommonV0Func::ingest")
trace_ingest_item!(
"TimeBinner for {} ingest common_range_current {:?} item {:?}",
self_name,
binner.common_range_current(),
item
);
if item.len() == 0 {
// Return already here, RangeOverlapInfo would not give much sense.
return;
}
// TODO optimize by remembering at which event array index we have arrived.
// That needs modified interfaces which can take and yield the start and latest index.
// Or consume the input data.
if B::common_has_lst(binner) == false {
if let Some(item) = item
.as_any_mut()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_mut::<B::Input>()
{
B::common_feed_lst(binner, item);
} else {
error!(
"{self_name}::ingest unexpected item type {} expected {}",
item.type_name(),
any::type_name::<B::Input>()
);
return;
}
}
loop {
while item.starts_after(B::common_range_current(binner)) {
trace_ingest_item!("{self_name} ignore item and cycle starts_after");
TimeBinnerCommonV0Func::cycle(binner);
if !B::common_has_more_range(binner) {
debug!("{self_name} no more bin in edges after starts_after");
return;
}
}
if item.ends_before(B::common_range_current(binner)) {
trace_ingest_item!("{self_name} ignore item ends_before");
return;
}
{
if !B::common_has_more_range(binner) {
trace_ingest_item!("{self_name} no more bin in edges");
return;
} else {
if let Some(item) = item
.as_any_mut()
// TODO make statically sure that we attempt to cast to the correct type here:
.downcast_mut::<B::Input>()
{
// TODO collect statistics associated with this request:
trace_ingest_item!("{self_name} FEED THE ITEM...");
TimeBinnerCommonV0Func::agg_ingest(binner, item);
if item.ends_after(B::common_range_current(binner)) {
trace_ingest_item!(
"{self_name} FED ITEM, ENDS AFTER agg-range {:?}",
B::common_range_current(binner)
);
TimeBinnerCommonV0Func::cycle(binner);
if !B::common_has_more_range(binner) {
warn!("{self_name} no more bin in edges after ingest and cycle");
return;
} else {
trace_ingest_item!("{self_name} item fed, cycled, continue");
}
} else {
trace_ingest_item!("{self_name} item fed, break");
break;
}
} else {
error!(
"{self_name}::ingest unexpected item type {} expected {}",
item.type_name(),
any::type_name::<B::Input>()
);
return;
};
}
}
}
} }
fn agg_ingest<B>(binner: &mut B, item: &mut <B as TimeBinnerCommonV0Trait>::Input) fn agg_ingest<B>(binner: &mut B, item: &mut <B as TimeBinnerCommonV0Trait>::Input)
@@ -243,7 +160,7 @@ impl ChooseIndicesForTimeBinEvents {
} }
pub trait TimeAggregatorCommonV0Trait { pub trait TimeAggregatorCommonV0Trait {
type Input: WithLen + RangeOverlapInfo + ChooseIndicesForTimeBin + 'static; type Input: WithLen + ChooseIndicesForTimeBin + 'static;
type Output: WithLen + Empty + AppendEmptyBin + HasNonemptyFirstBin + 'static; type Output: WithLen + Empty + AppendEmptyBin + HasNonemptyFirstBin + 'static;
fn type_name() -> &'static str; fn type_name() -> &'static str;
fn common_range_current(&self) -> &SeriesRange; fn common_range_current(&self) -> &SeriesRange;
crates/streams/src/timebinnedjson.rs
+60 -252
View File
@@ -8,13 +8,9 @@ use crate::tcprawclient::make_sub_query;
use crate::tcprawclient::OpenBoxedBytesStreamsBox; use crate::tcprawclient::OpenBoxedBytesStreamsBox;
use crate::timebin::cached::reader::EventsReadProvider; use crate::timebin::cached::reader::EventsReadProvider;
use crate::timebin::CacheReadProvider; use crate::timebin::CacheReadProvider;
use crate::timebin::TimeBinnedStream;
use crate::transform::build_merged_event_transform; use crate::transform::build_merged_event_transform;
use crate::transform::EventsToTimeBinnable;
use err::Error; use err::Error;
use futures_util::future::BoxFuture; use futures_util::future::BoxFuture;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream; use futures_util::Stream;
use futures_util::StreamExt; use futures_util::StreamExt;
use items_0::collect_s::CollectableDyn; use items_0::collect_s::CollectableDyn;
@@ -22,15 +18,9 @@ use items_0::on_sitemty_data;
use items_0::streamitem::RangeCompletableItem; use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty; use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem; use items_0::streamitem::StreamItem;
use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
use items_0::transform::TimeBinnableStreamBox;
use items_0::transform::TimeBinnableStreamTrait;
use items_0::Events; use items_0::Events;
use items_2::channelevents::ChannelEvents; use items_2::channelevents::ChannelEvents;
use items_2::merger::Merger; use items_2::merger::Merger;
use items_2::streams::PlainEventStream;
use netpod::log::*;
use netpod::log::*; use netpod::log::*;
use netpod::range::evrange::NanoRange; use netpod::range::evrange::NanoRange;
use netpod::BinnedRangeEnum; use netpod::BinnedRangeEnum;
@@ -43,8 +33,6 @@ use query::transform::TransformQuery;
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use std::pin::Pin; use std::pin::Pin;
use std::sync::Arc; use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
use std::time::Duration; use std::time::Duration;
use std::time::Instant; use std::time::Instant;
@@ -53,7 +41,7 @@ fn assert_stream_send<'u, R>(stream: impl 'u + Send + Stream<Item = R>) -> impl
stream stream
} }
pub async fn timebinnable_stream_sf_databuffer_box_events( pub async fn timebinnable_stream_sf_databuffer_channelevents(
range: NanoRange, range: NanoRange,
one_before_range: bool, one_before_range: bool,
ch_conf: ChannelTypeConfigGen, ch_conf: ChannelTypeConfigGen,
@@ -62,7 +50,7 @@ pub async fn timebinnable_stream_sf_databuffer_box_events(
log_level: String, log_level: String,
ctx: Arc<ReqCtx>, ctx: Arc<ReqCtx>,
open_bytes: OpenBoxedBytesStreamsBox, open_bytes: OpenBoxedBytesStreamsBox,
) -> Result<impl Stream<Item = Sitemty<Box<dyn Events>>>, Error> { ) -> Result<impl Stream<Item = Sitemty<ChannelEvents>>, Error> {
let subq = make_sub_query( let subq = make_sub_query(
ch_conf, ch_conf,
range.clone().into(), range.clone().into(),
@@ -85,21 +73,26 @@ pub async fn timebinnable_stream_sf_databuffer_box_events(
// TODO propagate also the max-buf-len for the first stage event reader. // TODO propagate also the max-buf-len for the first stage event reader.
// TODO use a mixture of count and byte-size as threshold. // TODO use a mixture of count and byte-size as threshold.
let stream = Merger::new(inps, sub.merger_out_len_max()); let stream = Merger::new(inps, sub.merger_out_len_max());
let stream = RangeFilter2::new(stream, range, one_before_range); let stream = RangeFilter2::new(stream, range, one_before_range);
let stream = stream.map(move |k: Sitemty<ChannelEvents>| {
let stream = stream.map(move |k| { use ChannelEvents;
on_sitemty_data!(k, |k: Box<dyn Events>| { use RangeCompletableItem::*;
TODO; use StreamItem::*;
let k: Box<dyn Events> = Box::new(k); match k {
let k = k.to_dim0_f32_for_binning(); Ok(DataItem(Data(ChannelEvents::Events(k)))) => {
let k = tr.0.transform(k); // let k = k;
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))) // let k: Box<dyn Events> = Box::new(k);
}) let k = k.to_dim0_f32_for_binning();
let k = tr.0.transform(k);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(ChannelEvents::Events(
k,
))))
}
_ => k,
}
}); });
#[cfg(target_abi = "")] #[cfg(feature = "wasm_transform")]
#[cfg(wasm_transform)]
let stream = if let Some(wasmname) = wasm1 { let stream = if let Some(wasmname) = wasm1 {
debug!("make wasm transform"); debug!("make wasm transform");
use httpclient::url::Url; use httpclient::url::Url;
@@ -225,242 +218,61 @@ pub async fn timebinnable_stream_sf_databuffer_box_events(
Ok(stream) Ok(stream)
} }
async fn timebinnable_stream_sf_databuffer_binnable_box(
range: NanoRange,
one_before_range: bool,
ch_conf: ChannelTypeConfigGen,
transform_query: TransformQuery,
sub: EventsSubQuerySettings,
log_level: String,
ctx: Arc<ReqCtx>,
open_bytes: OpenBoxedBytesStreamsBox,
) -> Result<TimeBinnableStreamBox, Error> {
let stream = timebinnable_stream_sf_databuffer_box_events(
range,
one_before_range,
ch_conf,
transform_query,
sub,
log_level,
ctx,
open_bytes,
)
.await?;
let stream = PlainEventStream::new(stream);
// let stream = stream.map(|x| {
// on_sitemty_data!(x, |x| {
// let ret = x.binnable;
// ret
// })
// });
let stream = EventsToTimeBinnable::new(stream);
let stream = Box::pin(stream);
Ok(TimeBinnableStreamBox(stream))
}
pub async fn timebinnable_stream_sf_databuffer_channelevents(
range: NanoRange,
one_before_range: bool,
ch_conf: ChannelTypeConfigGen,
transform_query: TransformQuery,
sub: EventsSubQuerySettings,
log_level: String,
ctx: Arc<ReqCtx>,
open_bytes: OpenBoxedBytesStreamsBox,
) -> Result<impl Stream<Item = Sitemty<ChannelEvents>>, Error> {
let stream = timebinnable_stream_sf_databuffer_box_events(
range,
one_before_range,
ch_conf,
transform_query,
sub,
log_level,
ctx,
open_bytes,
)
.await?;
// let stream = stream.map(|x| x);
let stream = stream.map(move |k| {
on_sitemty_data!(k, |k| {
// let k: Box<dyn Collectable> = Box::new(k);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(ChannelEvents::Events(
k,
))))
})
});
Ok(stream)
}
pub struct TimeBinnableStream {
make_stream_fut: Option<Pin<Box<dyn Future<Output = Result<TimeBinnableStreamBox, Error>> + Send>>>,
stream: Option<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinnable>>> + Send>>>,
}
impl TimeBinnableStream {
pub fn new(
range: NanoRange,
one_before_range: bool,
ch_conf: ChannelTypeConfigGen,
transform_query: TransformQuery,
sub: EventsSubQuerySettings,
log_level: String,
// TODO take by Arc ref
ctx: Arc<ReqCtx>,
open_bytes: OpenBoxedBytesStreamsBox,
) -> Self {
let fut = timebinnable_stream_sf_databuffer_binnable_box(
range,
one_before_range,
ch_conf,
transform_query,
sub,
log_level,
ctx,
open_bytes,
);
let fut = Box::pin(fut);
Self {
make_stream_fut: Some(fut),
stream: None,
}
}
}
// impl WithTransformProperties + Send
impl Stream for TimeBinnableStream {
type Item = Sitemty<Box<dyn TimeBinnable>>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if let Some(fut) = self.make_stream_fut.as_mut() {
match fut.poll_unpin(cx) {
Ready(x) => match x {
Ok(x) => {
self.make_stream_fut = None;
self.stream = Some(Box::pin(x));
continue;
}
Err(e) => Ready(Some(Err(e))),
},
Pending => Pending,
}
} else if let Some(fut) = self.stream.as_mut() {
match fut.poll_next_unpin(cx) {
Ready(Some(x)) => Ready(Some(x)),
Ready(None) => {
self.stream = None;
Ready(None)
}
Pending => Pending,
}
} else {
Ready(None)
};
}
}
}
async fn timebinned_stream( async fn timebinned_stream(
query: BinnedQuery, query: BinnedQuery,
binned_range: BinnedRangeEnum, binned_range: BinnedRangeEnum,
ch_conf: ChannelTypeConfigGen, ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx, ctx: &ReqCtx,
open_bytes: OpenBoxedBytesStreamsBox,
cache_read_provider: Arc<dyn CacheReadProvider>, cache_read_provider: Arc<dyn CacheReadProvider>,
events_read_provider: Arc<dyn EventsReadProvider>, events_read_provider: Arc<dyn EventsReadProvider>,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn CollectableDyn>>> + Send>>, Error> { ) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn CollectableDyn>>> + Send>>, Error> {
use netpod::query::CacheUsage; use netpod::query::CacheUsage;
let cache_usage = query.cache_usage().unwrap_or(CacheUsage::Ignore); let cache_usage = query.cache_usage().unwrap_or(CacheUsage::Ignore);
match cache_usage.clone() { debug!("BINNING NEW METHOD");
CacheUsage::Use | CacheUsage::Recreate | CacheUsage::Ignore => { debug!(
debug!("BINNING NEW METHOD"); "timebinned_stream caching {:?} subgrids {:?}",
debug!( query,
"timebinned_stream caching {:?} subgrids {:?}", query.subgrids()
query, );
query.subgrids() let do_time_weight = true;
); let bin_len_layers = if let Some(subgrids) = query.subgrids() {
let do_time_weight = true; subgrids
let bin_len_layers = if let Some(subgrids) = query.subgrids() { .iter()
subgrids .map(|&x| DtMs::from_ms_u64(1000 * x.as_secs()))
.iter() .collect()
.map(|&x| DtMs::from_ms_u64(1000 * x.as_secs())) } else {
.collect() netpod::time_bin_len_cache_opts().to_vec()
} else { };
netpod::time_bin_len_cache_opts().to_vec() let stream = crate::timebin::TimeBinnedFromLayers::new(
}; ch_conf,
let stream = crate::timebin::TimeBinnedFromLayers::new( cache_usage,
ch_conf, query.transform().clone(),
cache_usage, EventsSubQuerySettings::from(&query),
query.transform().clone(), query.log_level().into(),
EventsSubQuerySettings::from(&query), Arc::new(ctx.clone()),
query.log_level().into(), binned_range.binned_range_time(),
Arc::new(ctx.clone()), do_time_weight,
binned_range.binned_range_time(), bin_len_layers,
do_time_weight, cache_read_provider,
bin_len_layers, events_read_provider,
cache_read_provider, )
events_read_provider, .map_err(Error::from_string)?;
) let stream = stream.map(|item| {
.map_err(Error::from_string)?; use items_0::timebin::BinningggContainerBinsDyn;
let stream = stream.map(|item| { on_sitemty_data!(item, |mut x: Box<dyn BinningggContainerBinsDyn>| {
use items_0::timebin::BinningggContainerBinsDyn; x.fix_numerics();
on_sitemty_data!(item, |mut x: Box<dyn BinningggContainerBinsDyn>| { let ret = Box::new(x) as Box<dyn CollectableDyn>;
x.fix_numerics(); Ok(StreamItem::DataItem(RangeCompletableItem::Data(ret)))
let ret = Box::new(x) as Box<dyn CollectableDyn>; })
Ok(StreamItem::DataItem(RangeCompletableItem::Data(ret))) });
}) let stream = Box::pin(stream);
}); Ok(stream)
let stream = Box::pin(stream);
Ok(stream)
}
_ => {
debug!("BINNING OLD METHOD");
let range = binned_range.binned_range_time().to_nano_range();
let do_time_weight = true;
let one_before_range = true;
let stream = timebinnable_stream_sf_databuffer_binnable_box(
range,
one_before_range,
ch_conf,
query.transform().clone(),
(&query).into(),
query.log_level().into(),
Arc::new(ctx.clone()),
open_bytes,
)
.await?;
let stream: Pin<Box<dyn TimeBinnableStreamTrait>> = stream.0;
let stream = Box::pin(stream);
// TODO rename TimeBinnedStream to make it more clear that it is the component which initiates the time binning.
let stream = TimeBinnedStream::new(stream, binned_range, do_time_weight);
if false {
let stream = stream.map(|x| {
on_sitemty_data!(x, |x: Box<dyn TimeBinned>| Ok(StreamItem::DataItem(
RangeCompletableItem::Data(x.to_container_bins())
)))
});
todo!();
}
let stream = stream.map(|x| {
on_sitemty_data!(x, |x| {
let ret = Box::new(x) as Box<dyn CollectableDyn>;
Ok(StreamItem::DataItem(RangeCompletableItem::Data(ret)))
})
});
// let stream: Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>> = Box::pin(stream);
let stream = Box::pin(stream);
Ok(stream)
}
}
} }
pub async fn timebinned_json( pub async fn timebinned_json(
query: BinnedQuery, query: BinnedQuery,
ch_conf: ChannelTypeConfigGen, ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx, ctx: &ReqCtx,
open_bytes: OpenBoxedBytesStreamsBox,
cache_read_provider: Arc<dyn CacheReadProvider>, cache_read_provider: Arc<dyn CacheReadProvider>,
events_read_provider: Arc<dyn EventsReadProvider>, events_read_provider: Arc<dyn EventsReadProvider>,
) -> Result<CollectResult<JsonValue>, Error> { ) -> Result<CollectResult<JsonValue>, Error> {
@@ -479,18 +291,16 @@ pub async fn timebinned_json(
binned_range.clone(), binned_range.clone(),
ch_conf, ch_conf,
ctx, ctx,
open_bytes,
cache_read_provider, cache_read_provider,
events_read_provider, events_read_provider,
) )
.await?; .await?;
// let stream = timebinned_to_collectable(stream);
let collected = Collect::new(stream, deadline, collect_max, bytes_max, None, Some(binned_range)); let collected = Collect::new(stream, deadline, collect_max, bytes_max, None, Some(binned_range));
let collected: BoxFuture<_> = Box::pin(collected); let collected: BoxFuture<_> = Box::pin(collected);
let collres = collected.await?; let collres = collected.await?;
match collres { match collres {
CollectResult::Some(collres) => { CollectResult::Some(collres) => {
let collres = if let Some(bins) = collres let collres = if let Some(_bins) = collres
.as_any_ref() .as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>() .downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{ {
@@ -511,7 +321,7 @@ pub async fn timebinned_json(
fn take_collector_result(coll: &mut Box<dyn items_0::collect_s::CollectorDyn>) -> Option<serde_json::Value> { fn take_collector_result(coll: &mut Box<dyn items_0::collect_s::CollectorDyn>) -> Option<serde_json::Value> {
match coll.result(None, None) { match coll.result(None, None) {
Ok(collres) => { Ok(collres) => {
let collres = if let Some(bins) = collres let collres = if let Some(_bins) = collres
.as_any_ref() .as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>() .downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{ {
@@ -534,7 +344,6 @@ pub async fn timebinned_json_framed(
query: BinnedQuery, query: BinnedQuery,
ch_conf: ChannelTypeConfigGen, ch_conf: ChannelTypeConfigGen,
ctx: &ReqCtx, ctx: &ReqCtx,
open_bytes: OpenBoxedBytesStreamsBox,
cache_read_provider: Arc<dyn CacheReadProvider>, cache_read_provider: Arc<dyn CacheReadProvider>,
events_read_provider: Arc<dyn EventsReadProvider>, events_read_provider: Arc<dyn EventsReadProvider>,
) -> Result<JsonStream, Error> { ) -> Result<JsonStream, Error> {
@@ -546,7 +355,6 @@ pub async fn timebinned_json_framed(
binned_range.clone(), binned_range.clone(),
ch_conf, ch_conf,
ctx, ctx,
open_bytes,
cache_read_provider, cache_read_provider,
events_read_provider, events_read_provider,
) )