daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP refactor data event pipeline

Browse Source
This commit is contained in:
Dominik Werder
2022-11-15 16:16:16 +01:00
parent fb78f1887e
commit eebf8665ce
24 changed files with 800 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
items_2/src/binsdim0.rs
View File

@@ -231,16 +231,14 @@ pub struct BinsDim0Collector<NTY> {
timed_out: bool,
range_complete: bool,
vals: BinsDim0<NTY>,
bin_count_exp: u32,
}
impl<NTY> BinsDim0Collector<NTY> {
pub fn new(bin_count_exp: u32) -> Self {
pub fn new() -> Self {
Self {
timed_out: false,
range_complete: false,
vals: BinsDim0::<NTY>::empty(),
bin_count_exp,
}
}
}
@@ -274,11 +272,12 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
}
fn result(&mut self) -> Result<Self::Output, Error> {
let bin_count_exp = 0;
let bin_count = self.vals.ts1s.len() as u32;
let (missing_bins, continue_at, finished_at) = if bin_count < self.bin_count_exp {
let (missing_bins, continue_at, finished_at) = if bin_count < bin_count_exp {
match self.vals.ts2s.back() {
Some(&k) => {
let missing_bins = self.bin_count_exp - bin_count;
let missing_bins = bin_count_exp - bin_count;
let continue_at = IsoDateTime(Utc.timestamp_nanos(k as i64));
let u = k + (k - self.vals.ts1s.back().unwrap()) * missing_bins as u64;
let finished_at = IsoDateTime(Utc.timestamp_nanos(u as i64));
@@ -323,8 +322,8 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
impl<NTY: ScalarOps> CollectableType for BinsDim0<NTY> {
type Collector = BinsDim0Collector<NTY>;
fn new_collector(bin_count_exp: u32) -> Self::Collector {
Self::Collector::new(bin_count_exp)
fn new_collector() -> Self::Collector {
Self::Collector::new()
}
}

9
items_2/src/eventsdim0.rs
View File

@@ -115,17 +115,14 @@ pub struct EventsDim0Collector<NTY> {
vals: EventsDim0<NTY>,
range_complete: bool,
timed_out: bool,
#[allow(unused)]
bin_count_exp: u32,
}
impl<NTY> EventsDim0Collector<NTY> {
pub fn new(bin_count_exp: u32) -> Self {
pub fn new() -> Self {
Self {
vals: EventsDim0::empty(),
range_complete: false,
timed_out: false,
bin_count_exp,
}
}
}
@@ -206,8 +203,8 @@ impl<NTY: ScalarOps> CollectorType for EventsDim0Collector<NTY> {
impl<NTY: ScalarOps> CollectableType for EventsDim0<NTY> {
type Collector = EventsDim0Collector<NTY>;
fn new_collector(bin_count_exp: u32) -> Self::Collector {
Self::Collector::new(bin_count_exp)
fn new_collector() -> Self::Collector {
Self::Collector::new()
}
}

88
items_2/src/items_2.rs
View File

@@ -310,6 +310,71 @@ impl PartialEq for Box<dyn Events> {
}
}
struct EventsCollector {}
impl WithLen for EventsCollector {
fn len(&self) -> usize {
todo!()
}
}
impl Collector for EventsCollector {
fn ingest(&mut self, src: &mut dyn Collectable) {
todo!()
}
fn set_range_complete(&mut self) {
todo!()
}
fn set_timed_out(&mut self) {
todo!()
}
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, err::Error> {
todo!()
}
}
impl Collectable for Box<dyn Events> {
fn new_collector(&self) -> Box<dyn Collector> {
Box::new(EventsCollector {})
}
fn as_any_mut(&mut self) -> &mut dyn Any {
todo!()
}
}
/*impl crate::streams::CollectorType for EventsCollector {
type Input = dyn Events;
type Output = Box<dyn Events>;
fn ingest(&mut self, src: &mut Self::Input) {
todo!()
}
fn set_range_complete(&mut self) {
todo!()
}
fn set_timed_out(&mut self) {
todo!()
}
fn result(&mut self) -> Result<Self::Output, err::Error> {
todo!()
}
}*/
/*impl crate::streams::CollectableType for dyn Events {
type Collector = EventsCollector;
fn new_collector() -> Self::Collector {
todo!()
}
}*/
/// Data in time-binned form.
pub trait TimeBinned: Any + TimeBinnable {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnable;
@@ -506,6 +571,10 @@ pub enum ChannelEvents {
Status(ConnStatusEvent),
}
impl FrameTypeInnerStatic for ChannelEvents {
const FRAME_TYPE_ID: u32 = items::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
}
impl Clone for ChannelEvents {
fn clone(&self) -> Self {
match self {
@@ -663,14 +732,10 @@ impl MergableEvents for ChannelEvents {
}
}
impl FrameTypeInnerStatic for ChannelEvents {
const FRAME_TYPE_ID: u32 = items::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
}
// TODO do this with some blanket impl:
impl Collectable for Box<dyn Collectable> {
fn new_collector(&self, bin_count_exp: u32) -> Box<dyn streams::Collector> {
Collectable::new_collector(self.as_ref(), bin_count_exp)
fn new_collector(&self) -> Box<dyn streams::Collector> {
Collectable::new_collector(self.as_ref())
}
fn as_any_mut(&mut self) -> &mut dyn Any {
@@ -681,7 +746,6 @@ impl Collectable for Box<dyn Collectable> {
fn flush_binned(
binner: &mut Box<dyn TimeBinner>,
coll: &mut Option<Box<dyn Collector>>,
bin_count_exp: u32,
force: bool,
) -> Result<(), Error> {
trace!("flush_binned bins_ready_count: {}", binner.bins_ready_count());
@@ -699,7 +763,7 @@ fn flush_binned(
Some(mut ready) => {
trace!("binned_collected ready {ready:?}");
if coll.is_none() {
*coll = Some(ready.as_collectable_mut().new_collector(bin_count_exp));
*coll = Some(ready.as_collectable_mut().new_collector());
}
let cl = coll.as_mut().unwrap();
cl.ingest(ready.as_collectable_mut());
@@ -728,6 +792,8 @@ pub async fn binned_collected(
let ts_edges_max = *edges.last().unwrap();
let deadline = Instant::now() + timeout;
let mut did_timeout = false;
// TODO use a trait to allow check of unfinished data [hcn2956jxhwsf]
#[allow(unused)]
let bin_count_exp = edges.len().max(2) as u32 - 1;
let do_time_weight = agg_kind.do_time_weighted();
// TODO maybe TimeBinner should take all ChannelEvents and handle this?
@@ -772,7 +838,7 @@ pub async fn binned_collected(
}
let binner = binner.as_mut().unwrap();
binner.ingest(events.as_time_binnable());
flush_binned(binner, &mut coll, bin_count_exp, false)?;
flush_binned(binner, &mut coll, false)?;
}
}
ChannelEvents::Status(item) => {
@@ -804,10 +870,10 @@ pub async fn binned_collected(
binner.cycle();
}
trace!("flush binned");
flush_binned(&mut binner, &mut coll, bin_count_exp, false)?;
flush_binned(&mut binner, &mut coll, false)?;
if coll.is_none() {
debug!("force a bin");
flush_binned(&mut binner, &mut coll, bin_count_exp, true)?;
flush_binned(&mut binner, &mut coll, true)?;
} else {
trace!("coll is already some");
}

13
items_2/src/streams.rs
View File

@@ -23,12 +23,11 @@ pub trait Collector: Send + Unpin + WithLen {
pub trait CollectableType {
type Collector: CollectorType<Input = Self>;
fn new_collector(bin_count_exp: u32) -> Self::Collector;
fn new_collector() -> Self::Collector;
}
pub trait Collectable: Any {
fn new_collector(&self, bin_count_exp: u32) -> Box<dyn Collector>;
fn new_collector(&self) -> Box<dyn Collector>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}
@@ -53,8 +52,8 @@ impl<T: CollectorType + 'static> Collector for T {
}
impl<T: CollectableType + 'static> Collectable for T {
fn new_collector(&self, bin_count_exp: u32) -> Box<dyn Collector> {
Box::new(T::new_collector(bin_count_exp)) as _
fn new_collector(&self) -> Box<dyn Collector> {
Box::new(T::new_collector()) as _
}
fn as_any_mut(&mut self) -> &mut dyn Any {
@@ -69,11 +68,13 @@ pub trait ToJsonBytes {
}
// TODO check usage of this trait
pub trait ToJsonResult: fmt::Debug + Send {
pub trait ToJsonResult: erased_serde::Serialize + fmt::Debug + Send {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error>;
fn as_any(&self) -> &dyn Any;
}
erased_serde::serialize_trait_object!(ToJsonResult);
impl ToJsonResult for serde_json::Value {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error> {
Ok(Box::new(self.clone()))

6
items_2/src/test.rs
View File

@@ -185,6 +185,8 @@ fn bin01() {
let mut coll = None;
let mut binner = None;
let edges: Vec<_> = (0..10).into_iter().map(|t| SEC * 10 * t).collect();
// TODO implement continue-at [hcn2956jxhwsf]
#[allow(unused)]
let bin_count_exp = (edges.len() - 1) as u32;
let do_time_weight = true;
while let Some(item) = stream.next().await {
@@ -207,7 +209,7 @@ fn bin01() {
Some(mut ready) => {
eprintln!("ready {ready:?}");
if coll.is_none() {
coll = Some(ready.as_collectable_mut().new_collector(bin_count_exp));
coll = Some(ready.as_collectable_mut().new_collector());
}
let cl = coll.as_mut().unwrap();
cl.ingest(ready.as_collectable_mut());
@@ -240,7 +242,7 @@ fn bin01() {
Some(mut ready) => {
eprintln!("ready {ready:?}");
if coll.is_none() {
coll = Some(ready.as_collectable_mut().new_collector(bin_count_exp));
coll = Some(ready.as_collectable_mut().new_collector());
}
let cl = coll.as_mut().unwrap();
cl.ingest(ready.as_collectable_mut());