daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up, collect with timeout

Browse Source
This commit is contained in:
Dominik Werder
2022-12-19 14:09:37 +01:00
parent 64233b0ccb
commit 646ec38b3c
32 changed files with 622 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
items_2/src/binsdim0.rs
View File

@@ -10,9 +10,9 @@ use items_0::TimeBins;
use items_0::WithLen;
use items_0::{AppendEmptyBin, AsAnyRef};
use items_0::{AsAnyMut, Empty};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
use netpod::{log::*, BinnedRange};
use num_traits::Zero;
use serde::{Deserialize, Serialize};
use std::any::Any;
@@ -386,8 +386,12 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, Error> {
let bin_count_exp = 0;
fn result(&mut self, _range: Option<NanoRange>, binrange: Option<BinnedRange>) -> Result<Self::Output, Error> {
let bin_count_exp = if let Some(r) = &binrange {
r.bin_count() as u32
} else {
0
};
let bin_count = self.vals.ts1s.len() as u32;
let (missing_bins, continue_at, finished_at) = if bin_count < bin_count_exp {
match self.vals.ts2s.back() {
@@ -502,8 +506,12 @@ where
CollectorType::set_timed_out(self)
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
match CollectorType::result(self) {
fn result(
&mut self,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
match CollectorType::result(self, range, binrange) {
Ok(res) => Ok(Box::new(res)),
Err(e) => Err(e.into()),
}

18
items_2/src/binsxbindim0.rs
View File

@@ -10,9 +10,9 @@ use items_0::TimeBinned;
use items_0::TimeBins;
use items_0::WithLen;
use items_0::{AppendEmptyBin, AsAnyMut, AsAnyRef};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
use netpod::{log::*, BinnedRange};
use num_traits::Zero;
use serde::{Deserialize, Serialize};
use std::any::Any;
@@ -387,8 +387,12 @@ impl<NTY: ScalarOps> CollectorType for BinsXbinDim0Collector<NTY> {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, Error> {
let bin_count_exp = 0;
fn result(&mut self, _range: Option<NanoRange>, binrange: Option<BinnedRange>) -> Result<Self::Output, Error> {
let bin_count_exp = if let Some(r) = &binrange {
r.bin_count() as u32
} else {
0
};
let bin_count = self.vals.ts1s.len() as u32;
let (missing_bins, continue_at, finished_at) = if bin_count < bin_count_exp {
match self.vals.ts2s.back() {
@@ -503,8 +507,12 @@ where
CollectorType::set_timed_out(self)
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
match CollectorType::result(self) {
fn result(
&mut self,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
match CollectorType::result(self, range, binrange) {
Ok(res) => Ok(Box::new(res)),
Err(e) => Err(e.into()),
}

23
items_2/src/channelevents.rs
View File

@@ -8,6 +8,8 @@ use items_0::collect_s::Collector;
use items_0::AsAnyMut;
use items_0::AsAnyRef;
use netpod::log::*;
use netpod::BinnedRange;
use netpod::NanoRange;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::fmt;
@@ -669,7 +671,11 @@ impl items_0::collect_c::Collector for ChannelEventsCollector {
self.timed_out = true;
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
fn result(
&mut self,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
match self.coll.as_mut() {
Some(coll) => {
if self.range_complete {
@@ -678,20 +684,25 @@ impl items_0::collect_c::Collector for ChannelEventsCollector {
if self.timed_out {
coll.set_timed_out();
}
let res = coll.result()?;
//error!("fix output of ChannelEventsCollector [03ce6bc5a]");
//err::todo();
//let output = ChannelEventsCollectorOutput {};
let res = coll.result(range, binrange)?;
Ok(res)
}
None => {
error!("nothing collected [caa8d2565]");
todo!()
Err(err::Error::with_public_msg_no_trace("nothing collected [caa8d2565]"))
}
}
}
}
impl items_0::WithLen for ChannelEvents {
fn len(&self) -> usize {
match self {
ChannelEvents::Events(k) => k.len(),
ChannelEvents::Status(_) => 1,
}
}
}
impl items_0::collect_c::Collectable for ChannelEvents {
fn new_collector(&self) -> Box<dyn items_0::collect_c::Collector> {
Box::new(ChannelEventsCollector::new())

54
items_2/src/eventsdim0.rs
View File

@@ -5,9 +5,9 @@ use crate::{TimeBinnable, TimeBinnableType, TimeBinnableTypeAggregator, TimeBinn
use err::Error;
use items_0::scalar_ops::ScalarOps;
use items_0::{AsAnyMut, AsAnyRef, Empty, Events, WithLen};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
use netpod::{log::*, BinnedRange};
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::VecDeque;
@@ -151,7 +151,7 @@ where
#[derive(Debug)]
pub struct EventsDim0Collector<NTY> {
vals: EventsDim0<NTY>,
range_complete: bool,
range_final: bool,
timed_out: bool,
}
@@ -159,7 +159,7 @@ impl<NTY> EventsDim0Collector<NTY> {
pub fn new() -> Self {
Self {
vals: EventsDim0::empty(),
range_complete: false,
range_final: false,
timed_out: false,
}
}
@@ -186,7 +186,7 @@ pub struct EventsDim0CollectorOutput<NTY> {
#[serde(rename = "values")]
values: VecDeque<NTY>,
#[serde(rename = "rangeFinal", default, skip_serializing_if = "crate::bool_is_false")]
range_complete: bool,
range_final: bool,
#[serde(rename = "timedOut", default, skip_serializing_if = "crate::bool_is_false")]
timed_out: bool,
#[serde(rename = "continueAt", default, skip_serializing_if = "Option::is_none")]
@@ -220,7 +220,7 @@ impl<NTY: ScalarOps> EventsDim0CollectorOutput<NTY> {
}
pub fn range_complete(&self) -> bool {
self.range_complete
self.range_final
}
pub fn timed_out(&self) -> bool {
@@ -266,14 +266,14 @@ impl<NTY: ScalarOps> items_0::collect_s::CollectorType for EventsDim0Collector<N
}
fn set_range_complete(&mut self) {
self.range_complete = true;
self.range_final = true;
}
fn set_timed_out(&mut self) {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, Error> {
fn result(&mut self, range: Option<NanoRange>, _binrange: Option<BinnedRange>) -> Result<Self::Output, Error> {
// If we timed out, we want to hint the client from where to continue.
// This is tricky: currently, client can not request a left-exclusive range.
// We currently give the timestamp of the last event plus a small delta.
@@ -281,13 +281,17 @@ impl<NTY: ScalarOps> items_0::collect_s::CollectorType for EventsDim0Collector<N
// can parse and handle.
let continue_at = if self.timed_out {
if let Some(ts) = self.vals.tss.back() {
Some(IsoDateTime::from_u64(*ts))
Some(IsoDateTime::from_u64(*ts + netpod::timeunits::MS))
} else {
// TODO tricky: should yield again the original range begin? Leads to recursion.
// Range begin plus delta?
// Anyway, we don't have the range begin here.
warn!("timed out without any result, can not yield a continue-at");
None
if let Some(range) = &range {
Some(IsoDateTime::from_u64(range.beg + netpod::timeunits::SEC))
} else {
// TODO tricky: should yield again the original range begin? Leads to recursion.
// Range begin plus delta?
// Anyway, we don't have the range begin here.
warn!("timed out without any result, can not yield a continue-at");
None
}
}
} else {
None
@@ -303,7 +307,7 @@ impl<NTY: ScalarOps> items_0::collect_s::CollectorType for EventsDim0Collector<N
pulse_anchor,
pulse_off: pulse_off,
values: mem::replace(&mut self.vals.values, VecDeque::new()),
range_complete: self.range_complete,
range_final: self.range_final,
timed_out: self.timed_out,
continue_at,
};
@@ -340,8 +344,12 @@ impl<NTY: ScalarOps> items_0::collect_c::Collector for EventsDim0Collector<NTY>
items_0::collect_s::CollectorType::set_timed_out(self)
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
match items_0::collect_s::CollectorType::result(self) {
fn result(
&mut self,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
match items_0::collect_s::CollectorType::result(self, range, binrange) {
Ok(x) => Ok(Box::new(x)),
Err(e) => Err(e.into()),
}
@@ -970,7 +978,11 @@ impl items_0::collect_c::CollectorDyn for EventsDim0CollectorDyn {
todo!()
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
fn result(
&mut self,
_range: Option<NanoRange>,
_binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
todo!()
}
}
@@ -998,8 +1010,12 @@ impl<NTY: ScalarOps> items_0::collect_c::CollectorDyn for EventsDim0Collector<NT
items_0::collect_s::CollectorType::set_timed_out(self);
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
items_0::collect_s::CollectorType::result(self)
fn result(
&mut self,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
items_0::collect_s::CollectorType::result(self, range, binrange)
.map(|x| Box::new(x) as _)
.map_err(|e| e.into())
}

43
items_2/src/eventsxbindim0.rs
View File

@@ -1,13 +1,13 @@
use crate::binsxbindim0::BinsXbinDim0;
use crate::RangeOverlapInfo;
use crate::{pulse_offs_from_abs, ts_offs_from_abs};
use crate::{IsoDateTime, RangeOverlapInfo};
use crate::{TimeBinnableType, TimeBinnableTypeAggregator};
use err::Error;
use items_0::scalar_ops::ScalarOps;
use items_0::{AsAnyMut, WithLen};
use items_0::{AsAnyRef, Empty};
use netpod::log::*;
use netpod::NanoRange;
use netpod::{log::*, BinnedRange};
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::VecDeque;
@@ -359,10 +359,11 @@ pub struct EventsXbinDim0CollectorOutput<NTY> {
#[serde(rename = "avgs")]
avgs: VecDeque<f32>,
#[serde(rename = "rangeFinal", default, skip_serializing_if = "crate::bool_is_false")]
finalised_range: bool,
range_final: bool,
#[serde(rename = "timedOut", default, skip_serializing_if = "crate::bool_is_false")]
timed_out: bool,
// TODO add continue-at
#[serde(rename = "continueAt", default, skip_serializing_if = "Option::is_none")]
continue_at: Option<IsoDateTime>,
}
impl<NTY> AsAnyRef for EventsXbinDim0CollectorOutput<NTY>
@@ -398,14 +399,14 @@ impl<NTY> items_0::collect_c::Collected for EventsXbinDim0CollectorOutput<NTY> w
#[derive(Debug)]
pub struct EventsXbinDim0Collector<NTY> {
vals: EventsXbinDim0<NTY>,
finalised_range: bool,
range_final: bool,
timed_out: bool,
}
impl<NTY> EventsXbinDim0Collector<NTY> {
pub fn new() -> Self {
Self {
finalised_range: false,
range_final: false,
timed_out: false,
vals: EventsXbinDim0::empty(),
}
@@ -434,15 +435,32 @@ where
}
fn set_range_complete(&mut self) {
self.finalised_range = true;
self.range_final = true;
}
fn set_timed_out(&mut self) {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, Error> {
fn result(&mut self, range: Option<NanoRange>, _binrange: Option<BinnedRange>) -> Result<Self::Output, Error> {
use std::mem::replace;
let continue_at = if self.timed_out {
if let Some(ts) = self.vals.tss.back() {
Some(IsoDateTime::from_u64(*ts + netpod::timeunits::MS))
} else {
if let Some(range) = &range {
Some(IsoDateTime::from_u64(range.beg + netpod::timeunits::SEC))
} else {
// TODO tricky: should yield again the original range begin? Leads to recursion.
// Range begin plus delta?
// Anyway, we don't have the range begin here.
warn!("timed out without any result, can not yield a continue-at");
None
}
}
} else {
None
};
let mins = replace(&mut self.vals.mins, VecDeque::new());
let maxs = replace(&mut self.vals.maxs, VecDeque::new());
let avgs = replace(&mut self.vals.avgs, VecDeque::new());
@@ -459,8 +477,9 @@ where
mins,
maxs,
avgs,
finalised_range: self.finalised_range,
range_final: self.range_final,
timed_out: self.timed_out,
continue_at,
};
Ok(ret)
}
@@ -497,7 +516,11 @@ where
todo!()
}
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
fn result(
&mut self,
_range: Option<NanoRange>,
_binrange: Option<BinnedRange>,
) -> Result<Box<dyn items_0::collect_c::Collected>, Error> {
todo!()
}
}

5
items_2/src/items_2.rs
View File

@@ -371,7 +371,8 @@ fn flush_binned(
}
}
// TODO remove
// TODO remove.
// Compare with items_2::test::bin01
pub async fn binned_collected(
scalar_type: ScalarType,
shape: Shape,
@@ -477,7 +478,7 @@ pub async fn binned_collected(
}
match coll {
Some(mut coll) => {
let res = coll.result().map_err(|e| format!("{e}"))?;
let res = coll.result(None, None).map_err(|e| format!("{e}"))?;
tokio::time::sleep(Duration::from_millis(2000)).await;
Ok(res)
}

9
items_2/src/test.rs
View File

@@ -298,7 +298,12 @@ fn bin01() {
let mut stream = ChannelEventsMerger::new(vec![inp1, inp2]);
let mut coll = None;
let mut binner = None;
let edges: Vec<_> = (0..10).into_iter().map(|t| SEC * 10 * t).collect();
let range = NanoRange {
beg: SEC * 0,
end: SEC * 100,
};
let binrange = BinnedRange::covering_range(range, 10).unwrap();
let edges = binrange.edges();
// TODO implement continue-at [hcn2956jxhwsf]
#[allow(unused)]
let bin_count_exp = (edges.len() - 1) as u32;
@@ -369,7 +374,7 @@ fn bin01() {
}
match coll {
Some(mut coll) => {
let res = coll.result().map_err(|e| format!("{e}"))?;
let res = coll.result(None, Some(binrange.clone())).map_err(|e| format!("{e}"))?;
//let res = res.to_json_result().map_err(|e| format!("{e}"))?;
//let res = res.to_json_bytes().map_err(|e| format!("{e}"))?;
eprintln!("res {res:?}");