factor out into common crate

This commit is contained in:
Dominik Werder
2022-11-29 15:50:16 +01:00
parent 075bdd05bd
commit 9fe63706cf
25 changed files with 466 additions and 449 deletions

View File

@@ -29,6 +29,7 @@ dbconn = { path = "../dbconn" }
tokio-postgres = { version = "0.7.6", features = ["runtime", "with-chrono-0_4", "with-serde_json-1"] }
disk = { path = "../disk" }
items = { path = "../items" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }
parse = { path = "../parse" }
streams = { path = "../streams" }

View File

@@ -273,7 +273,7 @@ impl EventsHandlerScylla {
Ok(k) => match k {
ChannelEvents::Events(mut item) => {
if coll.is_none() {
coll = Some(items_2::streams::Collectable::new_collector(item.as_ref()));
coll = Some(items_0::collect_s::Collectable::new_collector(item.as_ref()));
}
let cl = coll.as_mut().unwrap();
cl.ingest(item.as_collectable_mut());

View File

@@ -23,5 +23,6 @@ chrono = { version = "0.4.22", features = ["serde"] }
crc32fast = "1.3.2"
err = { path = "../err" }
items_proc = { path = "../items_proc" }
items_0 = { path = "../items_0" }
netpod = { path = "../netpod" }
parse = { path = "../parse" }

15
items_0/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "items_0"
version = "0.0.1"
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2021"
[lib]
path = "src/items_0.rs"
[dependencies]
serde = { version = "1.0", features = ["derive"] }
erased-serde = "0.3"
serde_json = "1.0"
netpod = { path = "../netpod" }
err = { path = "../err" }

118
items_0/src/collect_c.rs Normal file
View File

@@ -0,0 +1,118 @@
use crate::collect_s::ToJsonBytes;
use crate::collect_s::ToJsonResult;
use crate::AsAnyRef;
use crate::Events;
use err::Error;
use std::any::Any;
use std::fmt;
pub trait Collector: fmt::Debug + Send {
// TODO should require here Collectable?
type Input;
type Output: Collected;
fn len(&self) -> usize;
fn ingest(&mut self, item: &mut Self::Input);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Self::Output, Error>;
}
pub trait Collectable: fmt::Debug {
type Collector: Collector<Input = Self>;
fn new_collector(&self) -> Self::Collector;
}
pub trait Collected: fmt::Debug + ToJsonResult + AsAnyRef + Send {}
erased_serde::serialize_trait_object!(Collected);
impl AsAnyRef for Box<dyn Collected> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}
impl ToJsonResult for Box<dyn Collected> {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error> {
self.as_ref().to_json_result()
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl Collected for Box<dyn Collected> {}
#[derive(Debug)]
pub struct CollectorDynDefault {}
pub trait CollectorDyn: fmt::Debug + Send {
fn len(&self) -> usize;
fn ingest(&mut self, item: &mut dyn CollectableWithDefault);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Box<dyn Collected>, Error>;
}
pub trait CollectableWithDefault {
fn new_collector(&self) -> Box<dyn CollectorDyn>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}
#[derive(Debug)]
pub struct EventsCollector {
coll: Box<dyn CollectorDyn>,
}
impl EventsCollector {
pub fn new(coll: Box<dyn CollectorDyn>) -> Self {
Self { coll }
}
}
impl Collector for EventsCollector {
type Input = Box<dyn Events>;
// TODO this Output trait does not differentiate between e.g. collected events, collected bins, different aggs, etc...
type Output = Box<dyn Collected>;
fn len(&self) -> usize {
self.coll.len()
}
fn ingest(&mut self, item: &mut Self::Input) {
self.coll.ingest(item.as_collectable_with_default_mut());
}
fn set_range_complete(&mut self) {
self.coll.set_range_complete()
}
fn set_timed_out(&mut self) {
self.coll.set_timed_out()
}
fn result(&mut self) -> Result<Self::Output, Error> {
self.coll.result()
}
}
impl Collectable for Box<dyn Events> {
type Collector = EventsCollector;
fn new_collector(&self) -> Self::Collector {
let coll = CollectableWithDefault::new_collector(self.as_ref());
EventsCollector::new(coll)
}
}

106
items_0/src/collect_s.rs Normal file
View File

@@ -0,0 +1,106 @@
use super::collect_c::Collected;
use crate::WithLen;
use err::Error;
use serde::Serialize;
use std::any::Any;
use std::fmt;
pub trait CollectorType: Send + Unpin + WithLen {
type Input: Collectable;
type Output: Collected + ToJsonResult + Serialize;
fn ingest(&mut self, src: &mut Self::Input);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
// TODO use this crate's Error instead:
fn result(&mut self) -> Result<Self::Output, Error>;
}
pub trait Collector: Send + Unpin + WithLen {
fn ingest(&mut self, src: &mut dyn Collectable);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, Error>;
}
pub trait CollectableType {
type Collector: CollectorType<Input = Self>;
fn new_collector() -> Self::Collector;
}
pub trait Collectable: Any {
fn new_collector(&self) -> Box<dyn Collector>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}
impl<T: CollectorType + 'static> Collector for T {
fn ingest(&mut self, src: &mut dyn Collectable) {
let src: &mut <T as CollectorType>::Input = src.as_any_mut().downcast_mut().expect("can not downcast");
T::ingest(self, src)
}
fn set_range_complete(&mut self) {
T::set_range_complete(self)
}
fn set_timed_out(&mut self) {
T::set_timed_out(self)
}
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, Error> {
let ret = T::result(self)?;
Ok(Box::new(ret) as _)
}
}
impl<T: CollectableType + 'static> Collectable for T {
fn new_collector(&self) -> Box<dyn Collector> {
Box::new(T::new_collector()) as _
}
fn as_any_mut(&mut self) -> &mut dyn Any {
// TODO interesting: why exactly does returning `&mut self` not work here?
self
}
}
// TODO check usage of this trait
pub trait ToJsonBytes {
fn to_json_bytes(&self) -> Result<Vec<u8>, Error>;
}
// TODO check usage of this trait
pub trait ToJsonResult: erased_serde::Serialize + fmt::Debug + Send {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error>;
fn as_any(&self) -> &dyn Any;
}
erased_serde::serialize_trait_object!(ToJsonResult);
impl ToJsonResult for serde_json::Value {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error> {
Ok(Box::new(self.clone()))
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl ToJsonBytes for serde_json::Value {
fn to_json_bytes(&self) -> Result<Vec<u8>, Error> {
Ok(serde_json::to_vec(self)?)
}
}
// TODO do this with some blanket impl:
impl Collectable for Box<dyn Collectable> {
fn new_collector(&self) -> Box<dyn Collector> {
Collectable::new_collector(self.as_ref())
}
fn as_any_mut(&mut self) -> &mut dyn Any {
Collectable::as_any_mut(self.as_mut())
}
}

139
items_0/src/items_0.rs Normal file
View File

@@ -0,0 +1,139 @@
pub mod collect_c;
pub mod collect_s;
use collect_c::CollectableWithDefault;
use collect_s::Collectable;
use collect_s::ToJsonResult;
use netpod::{NanoRange, ScalarType, Shape};
use std::any::Any;
use std::fmt;
pub trait WithLen {
fn len(&self) -> usize;
}
// TODO can probably be removed.
pub trait TimeBins {
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn ts_min_max(&self) -> Option<(u64, u64)>;
}
pub enum Fits {
Empty,
Lower,
Greater,
Inside,
PartlyLower,
PartlyGreater,
PartlyLowerAndGreater,
}
pub trait RangeOverlapInfo {
fn ends_before(&self, range: NanoRange) -> bool;
fn ends_after(&self, range: NanoRange) -> bool;
fn starts_after(&self, range: NanoRange) -> bool;
}
pub trait EmptyForScalarTypeShape {
fn empty(scalar_type: ScalarType, shape: Shape) -> Self;
}
pub trait EmptyForShape {
fn empty(shape: Shape) -> Self;
}
pub trait Empty {
fn empty() -> Self;
}
pub trait AppendEmptyBin {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64);
}
pub trait AsAnyRef {
fn as_any_ref(&self) -> &dyn Any;
}
pub trait AsAnyMut {
fn as_any_mut(&mut self) -> &mut dyn Any;
}
/*impl AsAnyRef for Box<dyn AsAnyRef> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}*/
/// Data in time-binned form.
pub trait TimeBinned: Any + TimeBinnable {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnable;
fn as_collectable_mut(&mut self) -> &mut dyn Collectable;
fn edges_slice(&self) -> (&[u64], &[u64]);
fn counts(&self) -> &[u64];
fn mins(&self) -> Vec<f32>;
fn maxs(&self) -> Vec<f32>;
fn avgs(&self) -> Vec<f32>;
fn validate(&self) -> Result<(), String>;
}
pub trait TimeBinner: Send {
fn ingest(&mut self, item: &dyn TimeBinnable);
fn bins_ready_count(&self) -> usize;
fn bins_ready(&mut self) -> Option<Box<dyn TimeBinned>>;
/// If there is a bin in progress with non-zero count, push it to the result set.
/// With push_empty == true, a bin in progress is pushed even if it contains no counts.
fn push_in_progress(&mut self, push_empty: bool);
/// Implies `Self::push_in_progress` but in addition, pushes a zero-count bin if the call
/// to `push_in_progress` did not change the result count, as long as edges are left.
/// The next call to `Self::bins_ready_count` must return one higher count than before.
fn cycle(&mut self);
fn set_range_complete(&mut self);
}
// TODO remove the Any bound. Factor out into custom AsAny trait.
/// Provides a time-binned representation of the implementing type.
/// In contrast to `TimeBinnableType` this is meant for trait objects.
pub trait TimeBinnable: fmt::Debug + WithLen + RangeOverlapInfo + Any + Send {
// TODO implementors may fail if edges contain not at least 2 entries.
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Box<dyn TimeBinner>;
fn as_any(&self) -> &dyn Any;
// TODO just a helper for the empty result.
fn to_box_to_json_result(&self) -> Box<dyn ToJsonResult>;
}
// TODO can I remove the Any bound?
/// Container of some form of events, for use as trait object.
pub trait Events:
fmt::Debug + Any + Collectable + CollectableWithDefault + TimeBinnable + Send + erased_serde::Serialize
{
fn as_time_binnable(&self) -> &dyn TimeBinnable;
fn verify(&self) -> bool;
fn output_info(&self);
fn as_collectable_mut(&mut self) -> &mut dyn Collectable;
fn as_collectable_with_default_ref(&self) -> &dyn CollectableWithDefault;
fn as_collectable_with_default_mut(&mut self) -> &mut dyn CollectableWithDefault;
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_fresh(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()>;
fn clone_dyn(&self) -> Box<dyn Events>;
fn partial_eq_dyn(&self, other: &dyn Events) -> bool;
fn serde_id(&self) -> &'static str;
fn nty_id(&self) -> u32;
}
erased_serde::serialize_trait_object!(Events);
impl PartialEq for Box<dyn Events> {
fn eq(&self, other: &Self) -> bool {
Events::partial_eq_dyn(self.as_ref(), other.as_ref())
}
}

View File

@@ -21,6 +21,7 @@ futures-util = "0.3.24"
tokio = { version = "1.20", features = ["rt-multi-thread", "sync", "time"] }
err = { path = "../err" }
items = { path = "../items" }
items_0 = { path = "../items_0" }
items_proc = { path = "../items_proc" }
netpod = { path = "../netpod" }
taskrun = { path = "../taskrun" }

View File

@@ -1,11 +1,14 @@
use crate::streams::{Collectable, CollectableType, CollectorType, ToJsonResult};
use crate::{
ts_offs_from_abs, ts_offs_from_abs_with_anchor, AppendEmptyBin, Empty, IsoDateTime, RangeOverlapInfo, ScalarOps,
TimeBins, WithLen,
};
use crate::{TimeBinnable, TimeBinnableType, TimeBinnableTypeAggregator, TimeBinned, TimeBinner};
use crate::{ts_offs_from_abs, ts_offs_from_abs_with_anchor};
use crate::{IsoDateTime, RangeOverlapInfo, ScalarOps};
use crate::{TimeBinnable, TimeBinnableType, TimeBinnableTypeAggregator, TimeBinner};
use chrono::{TimeZone, Utc};
use err::Error;
use items_0::collect_s::{Collectable, CollectableType, CollectorType, ToJsonResult};
use items_0::AppendEmptyBin;
use items_0::Empty;
use items_0::TimeBinned;
use items_0::TimeBins;
use items_0::WithLen;
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
@@ -229,13 +232,13 @@ pub struct BinsDim0CollectedResult<NTY> {
finished_at: Option<IsoDateTime>,
}
impl<NTY: ScalarOps> crate::AsAnyRef for BinsDim0CollectedResult<NTY> {
impl<NTY: ScalarOps> items_0::AsAnyRef for BinsDim0CollectedResult<NTY> {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl<NTY: ScalarOps> crate::collect::Collected for BinsDim0CollectedResult<NTY> {}
impl<NTY: ScalarOps> items_0::collect_c::Collected for BinsDim0CollectedResult<NTY> {}
impl<NTY> BinsDim0CollectedResult<NTY> {
pub fn ts_anchor_sec(&self) -> u64 {
@@ -264,7 +267,7 @@ impl<NTY> BinsDim0CollectedResult<NTY> {
}
impl<NTY: ScalarOps> ToJsonResult for BinsDim0CollectedResult<NTY> {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, Error> {
fn to_json_result(&self) -> Result<Box<dyn items_0::collect_s::ToJsonBytes>, Error> {
let k = serde_json::to_value(self)?;
Ok(Box::new(k))
}
@@ -574,7 +577,7 @@ impl<NTY: ScalarOps> TimeBinner for BinsDim0TimeBinner<NTY> {
}
}
fn bins_ready(&mut self) -> Option<Box<dyn crate::TimeBinned>> {
fn bins_ready(&mut self) -> Option<Box<dyn items_0::TimeBinned>> {
match self.ready.take() {
Some(k) => Some(Box::new(k)),
None => None,

View File

@@ -1,14 +1,14 @@
use std::any::Any;
use std::fmt;
use crate::merger;
use crate::merger_cev::MergeableCev;
use crate::streams::Collectable;
use crate::streams::Collector;
use crate::{merger, Events};
use crate::Events;
use items::FrameType;
use items::FrameTypeInnerStatic;
use items_0::collect_s::Collectable;
use items_0::collect_s::Collector;
use netpod::log::*;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::fmt;
// TODO maybe rename to ChannelStatus?
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -170,7 +170,7 @@ mod serde_channel_events {
mod test_channel_events_serde {
use super::ChannelEvents;
use crate::eventsdim0::EventsDim0;
use crate::Empty;
use items_0::Empty;
#[test]
fn channel_events() {
@@ -298,8 +298,8 @@ pub struct ChannelEventsTimeBinner {
}
impl fmt::Debug for ChannelEventsTimeBinner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ChannelEventsTimeBinner")
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt.debug_struct("ChannelEventsTimeBinner")
.field("conn_state", &self.conn_state)
.finish()
}
@@ -307,7 +307,7 @@ impl fmt::Debug for ChannelEventsTimeBinner {
impl crate::timebin::TimeBinner for ChannelEventsTimeBinner {
type Input = ChannelEvents;
type Output = Box<dyn crate::TimeBinned>;
type Output = Box<dyn items_0::TimeBinned>;
fn ingest(&mut self, item: &mut Self::Input) {
match item {
@@ -386,14 +386,14 @@ impl crate::timebin::TimeBinnable for ChannelEvents {
#[derive(Debug, Serialize, Deserialize)]
pub struct ChannelEventsCollectorOutput {}
impl crate::AsAnyRef for ChannelEventsCollectorOutput {
impl items_0::AsAnyRef for ChannelEventsCollectorOutput {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl crate::ToJsonResult for ChannelEventsCollectorOutput {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, err::Error> {
fn to_json_result(&self) -> Result<Box<dyn items_0::collect_s::ToJsonBytes>, err::Error> {
todo!()
}
@@ -402,11 +402,11 @@ impl crate::ToJsonResult for ChannelEventsCollectorOutput {
}
}
impl crate::collect::Collected for ChannelEventsCollectorOutput {}
impl items_0::collect_c::Collected for ChannelEventsCollectorOutput {}
#[derive(Debug)]
pub struct ChannelEventsCollector {
coll: Option<Box<dyn crate::collect::CollectorDyn>>,
coll: Option<Box<dyn items_0::collect_c::CollectorDyn>>,
range_complete: bool,
timed_out: bool,
}
@@ -421,9 +421,9 @@ impl ChannelEventsCollector {
}
}
impl crate::collect::Collector for ChannelEventsCollector {
impl items_0::collect_c::Collector for ChannelEventsCollector {
type Input = ChannelEvents;
type Output = Box<dyn crate::collect::Collected>;
type Output = Box<dyn items_0::collect_c::Collected>;
fn len(&self) -> usize {
match &self.coll {
@@ -456,7 +456,7 @@ impl crate::collect::Collector for ChannelEventsCollector {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, crate::Error> {
fn result(&mut self) -> Result<Self::Output, err::Error> {
match self.coll.as_mut() {
Some(coll) => {
if self.range_complete {
@@ -479,7 +479,7 @@ impl crate::collect::Collector for ChannelEventsCollector {
}
}
impl crate::collect::Collectable for ChannelEvents {
impl items_0::collect_c::Collectable for ChannelEvents {
type Collector = ChannelEventsCollector;
fn new_collector(&self) -> Self::Collector {

View File

@@ -1,68 +1 @@
use crate::AsAnyRef;
use crate::Error;
use std::any::Any;
use std::fmt;
pub trait Collector: fmt::Debug + Send {
// TODO should require here Collectable?
type Input;
type Output: Collected;
fn len(&self) -> usize;
fn ingest(&mut self, item: &mut Self::Input);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Self::Output, Error>;
}
pub trait Collectable: fmt::Debug {
type Collector: Collector<Input = Self>;
fn new_collector(&self) -> Self::Collector;
}
pub trait Collected: fmt::Debug + crate::streams::ToJsonResult + AsAnyRef + Send {}
erased_serde::serialize_trait_object!(Collected);
impl AsAnyRef for Box<dyn Collected> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}
impl crate::streams::ToJsonResult for Box<dyn Collected> {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, err::Error> {
self.as_ref().to_json_result()
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl Collected for Box<dyn Collected> {}
#[derive(Debug)]
pub struct CollectorDynDefault {}
pub trait CollectorDyn: fmt::Debug + Send {
fn len(&self) -> usize;
fn ingest(&mut self, item: &mut dyn CollectableWithDefault);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Box<dyn Collected>, Error>;
}
pub trait CollectableWithDefault {
fn new_collector(&self) -> Box<dyn CollectorDyn>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}

View File

@@ -1,9 +1,9 @@
use crate::binsdim0::BinsDim0;
use crate::streams::{CollectableType, CollectorType, ToJsonResult};
use crate::ScalarOps;
use crate::{pulse_offs_from_abs, ts_offs_from_abs, RangeOverlapInfo};
use crate::{Empty, Events, ScalarOps, WithLen};
use crate::{TimeBinnable, TimeBinnableType, TimeBinnableTypeAggregator, TimeBinner};
use err::Error;
use items_0::{Empty, Events, WithLen};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
@@ -170,16 +170,16 @@ impl<NTY: ScalarOps> EventsDim0CollectorOutput<NTY> {
}
}
impl<NTY: ScalarOps> crate::AsAnyRef for EventsDim0CollectorOutput<NTY> {
impl<NTY: ScalarOps> items_0::AsAnyRef for EventsDim0CollectorOutput<NTY> {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl<NTY: ScalarOps> crate::collect::Collected for EventsDim0CollectorOutput<NTY> {}
impl<NTY: ScalarOps> items_0::collect_c::Collected for EventsDim0CollectorOutput<NTY> {}
impl<NTY: ScalarOps> ToJsonResult for EventsDim0CollectorOutput<NTY> {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, Error> {
impl<NTY: ScalarOps> items_0::collect_s::ToJsonResult for EventsDim0CollectorOutput<NTY> {
fn to_json_result(&self) -> Result<Box<dyn items_0::collect_s::ToJsonBytes>, Error> {
let k = serde_json::to_value(self)?;
Ok(Box::new(k))
}
@@ -189,7 +189,7 @@ impl<NTY: ScalarOps> ToJsonResult for EventsDim0CollectorOutput<NTY> {
}
}
impl<NTY: ScalarOps> CollectorType for EventsDim0Collector<NTY> {
impl<NTY: ScalarOps> items_0::collect_s::CollectorType for EventsDim0Collector<NTY> {
type Input = EventsDim0<NTY>;
type Output = EventsDim0CollectorOutput<NTY>;
@@ -226,7 +226,7 @@ impl<NTY: ScalarOps> CollectorType for EventsDim0Collector<NTY> {
}
}
impl<NTY: ScalarOps> CollectableType for EventsDim0<NTY> {
impl<NTY: ScalarOps> items_0::collect_s::CollectableType for EventsDim0<NTY> {
type Collector = EventsDim0Collector<NTY>;
fn new_collector() -> Self::Collector {
@@ -234,7 +234,7 @@ impl<NTY: ScalarOps> CollectableType for EventsDim0<NTY> {
}
}
impl<NTY: ScalarOps> crate::collect::Collector for EventsDim0Collector<NTY> {
impl<NTY: ScalarOps> items_0::collect_c::Collector for EventsDim0Collector<NTY> {
type Input = EventsDim0<NTY>;
type Output = EventsDim0CollectorOutput<NTY>;
@@ -243,19 +243,19 @@ impl<NTY: ScalarOps> crate::collect::Collector for EventsDim0Collector<NTY> {
}
fn ingest(&mut self, item: &mut Self::Input) {
CollectorType::ingest(self, item)
items_0::collect_s::CollectorType::ingest(self, item)
}
fn set_range_complete(&mut self) {
CollectorType::set_range_complete(self)
items_0::collect_s::CollectorType::set_range_complete(self)
}
fn set_timed_out(&mut self) {
CollectorType::set_timed_out(self)
items_0::collect_s::CollectorType::set_timed_out(self)
}
fn result(&mut self) -> Result<Self::Output, crate::Error> {
CollectorType::result(self).map_err(Into::into)
fn result(&mut self) -> Result<Self::Output, err::Error> {
items_0::collect_s::CollectorType::result(self).map_err(Into::into)
}
}
@@ -538,7 +538,7 @@ impl<NTY: ScalarOps> TimeBinnable for EventsDim0<NTY> {
self as &dyn Any
}
fn to_box_to_json_result(&self) -> Box<dyn ToJsonResult> {
fn to_box_to_json_result(&self) -> Box<dyn items_0::collect_s::ToJsonResult> {
let k = serde_json::to_value(self).unwrap();
Box::new(k) as _
}
@@ -585,15 +585,15 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
}
}
fn as_collectable_mut(&mut self) -> &mut dyn crate::streams::Collectable {
fn as_collectable_mut(&mut self) -> &mut dyn items_0::collect_s::Collectable {
self
}
fn as_collectable_with_default_ref(&self) -> &dyn crate::collect::CollectableWithDefault {
fn as_collectable_with_default_ref(&self) -> &dyn items_0::collect_c::CollectableWithDefault {
self
}
fn as_collectable_with_default_mut(&mut self) -> &mut dyn crate::collect::CollectableWithDefault {
fn as_collectable_with_default_mut(&mut self) -> &mut dyn items_0::collect_c::CollectableWithDefault {
self
}
@@ -619,7 +619,7 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrealted traits. Simplify.
if let Some(tgt) = crate::streams::Collectable::as_any_mut(tgt.as_mut()).downcast_mut::<Self>() {
if let Some(tgt) = items_0::collect_s::Collectable::as_any_mut(tgt.as_mut()).downcast_mut::<Self>() {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
// TODO make it harder to forget new members when the struct may get modified in the future
@@ -718,7 +718,7 @@ impl<NTY: ScalarOps> TimeBinner for EventsDim0TimeBinner<NTY> {
}
}
fn bins_ready(&mut self) -> Option<Box<dyn crate::TimeBinned>> {
fn bins_ready(&mut self) -> Option<Box<dyn items_0::TimeBinned>> {
match self.ready.take() {
Some(k) => Some(Box::new(k)),
None => None,
@@ -867,12 +867,12 @@ impl EventsDim0CollectorDyn {
}
}
impl crate::collect::CollectorDyn for EventsDim0CollectorDyn {
impl items_0::collect_c::CollectorDyn for EventsDim0CollectorDyn {
fn len(&self) -> usize {
todo!()
}
fn ingest(&mut self, _item: &mut dyn crate::collect::CollectableWithDefault) {
fn ingest(&mut self, _item: &mut dyn items_0::collect_c::CollectableWithDefault) {
// TODO remove this struct?
todo!()
}
@@ -885,20 +885,20 @@ impl crate::collect::CollectorDyn for EventsDim0CollectorDyn {
todo!()
}
fn result(&mut self) -> Result<Box<dyn crate::collect::Collected>, crate::Error> {
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
todo!()
}
}
impl<NTY: ScalarOps> crate::collect::CollectorDyn for EventsDim0Collector<NTY> {
impl<NTY: ScalarOps> items_0::collect_c::CollectorDyn for EventsDim0Collector<NTY> {
fn len(&self) -> usize {
WithLen::len(self)
}
fn ingest(&mut self, item: &mut dyn crate::collect::CollectableWithDefault) {
fn ingest(&mut self, item: &mut dyn items_0::collect_c::CollectableWithDefault) {
let x = item.as_any_mut();
if let Some(item) = x.downcast_mut::<EventsDim0<NTY>>() {
CollectorType::ingest(self, item)
items_0::collect_s::CollectorType::ingest(self, item)
} else {
// TODO need possibility to return error
()
@@ -906,22 +906,22 @@ impl<NTY: ScalarOps> crate::collect::CollectorDyn for EventsDim0Collector<NTY> {
}
fn set_range_complete(&mut self) {
CollectorType::set_range_complete(self);
items_0::collect_s::CollectorType::set_range_complete(self);
}
fn set_timed_out(&mut self) {
CollectorType::set_timed_out(self);
items_0::collect_s::CollectorType::set_timed_out(self);
}
fn result(&mut self) -> Result<Box<dyn crate::collect::Collected>, crate::Error> {
CollectorType::result(self)
fn result(&mut self) -> Result<Box<dyn items_0::collect_c::Collected>, err::Error> {
items_0::collect_s::CollectorType::result(self)
.map(|x| Box::new(x) as _)
.map_err(|e| e.into())
}
}
impl<NTY: ScalarOps> crate::collect::CollectableWithDefault for EventsDim0<NTY> {
fn new_collector(&self) -> Box<dyn crate::collect::CollectorDyn> {
impl<NTY: ScalarOps> items_0::collect_c::CollectableWithDefault for EventsDim0<NTY> {
fn new_collector(&self) -> Box<dyn items_0::collect_c::CollectorDyn> {
let coll = EventsDim0Collector::<NTY>::new();
Box::new(coll)
}
@@ -931,7 +931,7 @@ impl<NTY: ScalarOps> crate::collect::CollectableWithDefault for EventsDim0<NTY>
}
}
impl<NTY: ScalarOps> crate::collect::Collectable for EventsDim0<NTY> {
impl<NTY: ScalarOps> items_0::collect_c::Collectable for EventsDim0<NTY> {
type Collector = EventsDim0Collector<NTY>;
fn new_collector(&self) -> Self::Collector {

View File

@@ -10,10 +10,6 @@ pub mod test;
pub mod testgen;
pub mod timebin;
use crate as items_2;
use crate::streams::Collectable;
use crate::streams::Collector;
use crate::streams::ToJsonResult;
use channelevents::ChannelEvents;
use chrono::{DateTime, TimeZone, Utc};
use futures_util::FutureExt;
@@ -23,11 +19,16 @@ use items::RangeCompletableItem;
use items::Sitemty;
use items::StreamItem;
use items::SubFrId;
use items_0::collect_s::Collector;
use items_0::collect_s::ToJsonResult;
use items_0::Empty;
use items_0::Events;
use items_0::RangeOverlapInfo;
use items_0::{TimeBinnable, TimeBinner};
use netpod::log::*;
use netpod::timeunits::*;
use netpod::{AggKind, NanoRange, ScalarType, Shape};
use serde::{Deserialize, Serialize, Serializer};
use std::any::Any;
use std::collections::VecDeque;
use std::fmt;
use std::pin::Pin;
@@ -220,63 +221,6 @@ impl serde::de::Error for Error {
}
}
pub trait WithLen {
fn len(&self) -> usize;
}
// TODO can probably be removed.
pub trait TimeBins {
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn ts_min_max(&self) -> Option<(u64, u64)>;
}
pub enum Fits {
Empty,
Lower,
Greater,
Inside,
PartlyLower,
PartlyGreater,
PartlyLowerAndGreater,
}
pub trait RangeOverlapInfo {
fn ends_before(&self, range: NanoRange) -> bool;
fn ends_after(&self, range: NanoRange) -> bool;
fn starts_after(&self, range: NanoRange) -> bool;
}
pub trait EmptyForScalarTypeShape {
fn empty(scalar_type: ScalarType, shape: Shape) -> Self;
}
pub trait EmptyForShape {
fn empty(shape: Shape) -> Self;
}
pub trait Empty {
fn empty() -> Self;
}
pub trait AppendEmptyBin {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64);
}
pub trait AsAnyRef {
fn as_any_ref(&self) -> &dyn Any;
}
pub trait AsAnyMut {
fn as_any_mut(&mut self) -> &mut dyn Any;
}
/*impl AsAnyRef for Box<dyn AsAnyRef> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}*/
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct IsoDateTime(DateTime<Utc>);
@@ -295,87 +239,6 @@ pub fn make_iso_ts(tss: &[u64]) -> Vec<IsoDateTime> {
.collect()
}
pub trait TimeBinner: Send {
fn ingest(&mut self, item: &dyn TimeBinnable);
fn bins_ready_count(&self) -> usize;
fn bins_ready(&mut self) -> Option<Box<dyn TimeBinned>>;
/// If there is a bin in progress with non-zero count, push it to the result set.
/// With push_empty == true, a bin in progress is pushed even if it contains no counts.
fn push_in_progress(&mut self, push_empty: bool);
/// Implies `Self::push_in_progress` but in addition, pushes a zero-count bin if the call
/// to `push_in_progress` did not change the result count, as long as edges are left.
/// The next call to `Self::bins_ready_count` must return one higher count than before.
fn cycle(&mut self);
fn set_range_complete(&mut self);
}
// TODO remove the Any bound. Factor out into custom AsAny trait.
/// Provides a time-binned representation of the implementing type.
/// In contrast to `TimeBinnableType` this is meant for trait objects.
pub trait TimeBinnable: fmt::Debug + WithLen + RangeOverlapInfo + Any + Send {
// TODO implementors may fail if edges contain not at least 2 entries.
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Box<dyn TimeBinner>;
fn as_any(&self) -> &dyn Any;
// TODO just a helper for the empty result.
fn to_box_to_json_result(&self) -> Box<dyn ToJsonResult>;
}
// Helper trait to bridge between impls of event containers during refactoring.
// TODO get rid when no longer needed.
pub trait IntoEvents {
fn into_events(self) -> Box<dyn Events>;
}
impl<NTY> IntoEvents for items::scalarevents::ScalarEvents<NTY>
where
NTY: ScalarOps,
{
fn into_events(self) -> Box<dyn Events> {
let ret = items_2::eventsdim0::EventsDim0::<NTY> {
tss: self.tss.into(),
pulses: self.pulses.into(),
values: self.values.into(),
};
Box::new(ret)
}
}
// TODO can I remove the Any bound?
/// Container of some form of events, for use as trait object.
pub trait Events:
fmt::Debug
+ Any
+ Collectable
+ items_2::collect::CollectableWithDefault
+ TimeBinnable
+ Send
+ erased_serde::Serialize
{
fn as_time_binnable(&self) -> &dyn TimeBinnable;
fn verify(&self) -> bool;
fn output_info(&self);
fn as_collectable_mut(&mut self) -> &mut dyn Collectable;
fn as_collectable_with_default_ref(&self) -> &dyn crate::collect::CollectableWithDefault;
fn as_collectable_with_default_mut(&mut self) -> &mut dyn crate::collect::CollectableWithDefault;
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_fresh(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()>;
fn clone_dyn(&self) -> Box<dyn Events>;
fn partial_eq_dyn(&self, other: &dyn Events) -> bool;
fn serde_id(&self) -> &'static str;
fn nty_id(&self) -> u32;
}
erased_serde::serialize_trait_object!(Events);
impl crate::merger::Mergeable for Box<dyn Events> {
fn len(&self) -> usize {
self.as_ref().len()
@@ -405,24 +268,6 @@ impl crate::merger::Mergeable for Box<dyn Events> {
}
}
impl PartialEq for Box<dyn Events> {
fn eq(&self, other: &Self) -> bool {
Events::partial_eq_dyn(self.as_ref(), other.as_ref())
}
}
/// Data in time-binned form.
pub trait TimeBinned: Any + TimeBinnable {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnable;
fn as_collectable_mut(&mut self) -> &mut dyn Collectable;
fn edges_slice(&self) -> (&[u64], &[u64]);
fn counts(&self) -> &[u64];
fn mins(&self) -> Vec<f32>;
fn maxs(&self) -> Vec<f32>;
fn avgs(&self) -> Vec<f32>;
fn validate(&self) -> Result<(), String>;
}
pub trait TimeBinnableType: Send + Unpin + RangeOverlapInfo {
type Output: TimeBinnableType;
type Aggregator: TimeBinnableTypeAggregator<Input = Self, Output = Self::Output> + Send + Unpin;
@@ -570,64 +415,6 @@ pub fn empty_binned_dyn(scalar_type: &ScalarType, shape: &Shape, agg_kind: &AggK
}
}
#[derive(Debug)]
pub struct EventsCollector {
coll: Box<dyn items_2::collect::CollectorDyn>,
}
impl EventsCollector {
pub fn new(coll: Box<dyn items_2::collect::CollectorDyn>) -> Self {
Self { coll }
}
}
impl items_2::collect::Collector for EventsCollector {
type Input = Box<dyn Events>;
// TODO this Output trait does not differentiate between e.g. collected events, collected bins, different aggs, etc...
type Output = Box<dyn items_2::collect::Collected>;
fn len(&self) -> usize {
self.coll.len()
}
fn ingest(&mut self, item: &mut Self::Input) {
self.coll.ingest(item.as_collectable_with_default_mut());
}
fn set_range_complete(&mut self) {
self.coll.set_range_complete()
}
fn set_timed_out(&mut self) {
self.coll.set_timed_out()
}
fn result(&mut self) -> Result<Self::Output, Error> {
self.coll.result()
}
}
impl items_2::collect::Collectable for Box<dyn Events> {
type Collector = EventsCollector;
fn new_collector(&self) -> Self::Collector {
let coll = items_2::collect::CollectableWithDefault::new_collector(self.as_ref());
EventsCollector::new(coll)
}
}
// TODO do this with some blanket impl:
impl Collectable for Box<dyn Collectable> {
fn new_collector(&self) -> Box<dyn streams::Collector> {
Collectable::new_collector(self.as_ref())
}
fn as_any_mut(&mut self) -> &mut dyn Any {
Collectable::as_any_mut(self.as_mut())
}
}
fn flush_binned(
binner: &mut Box<dyn TimeBinner>,
coll: &mut Option<Box<dyn Collector>>,

View File

@@ -1,94 +1 @@
use crate::WithLen;
use err::Error;
use serde::Serialize;
use std::any::Any;
use std::fmt;
pub trait CollectorType: Send + Unpin + WithLen {
type Input: Collectable;
type Output: crate::collect::Collected + ToJsonResult + Serialize;
fn ingest(&mut self, src: &mut Self::Input);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
// TODO use this crate's Error instead:
fn result(&mut self) -> Result<Self::Output, Error>;
}
pub trait Collector: Send + Unpin + WithLen {
fn ingest(&mut self, src: &mut dyn Collectable);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, Error>;
}
pub trait CollectableType {
type Collector: CollectorType<Input = Self>;
fn new_collector() -> Self::Collector;
}
pub trait Collectable: Any {
fn new_collector(&self) -> Box<dyn Collector>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}
impl<T: CollectorType + 'static> Collector for T {
fn ingest(&mut self, src: &mut dyn Collectable) {
let src: &mut <T as CollectorType>::Input = src.as_any_mut().downcast_mut().expect("can not downcast");
T::ingest(self, src)
}
fn set_range_complete(&mut self) {
T::set_range_complete(self)
}
fn set_timed_out(&mut self) {
T::set_timed_out(self)
}
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, Error> {
let ret = T::result(self)?;
Ok(Box::new(ret) as _)
}
}
impl<T: CollectableType + 'static> Collectable for T {
fn new_collector(&self) -> Box<dyn Collector> {
Box::new(T::new_collector()) as _
}
fn as_any_mut(&mut self) -> &mut dyn Any {
// TODO interesting: why exactly does returning `&mut self` not work here?
self
}
}
// TODO check usage of this trait
pub trait ToJsonBytes {
fn to_json_bytes(&self) -> Result<Vec<u8>, Error>;
}
// TODO check usage of this trait
pub trait ToJsonResult: erased_serde::Serialize + fmt::Debug + Send {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error>;
fn as_any(&self) -> &dyn Any;
}
erased_serde::serialize_trait_object!(ToJsonResult);
impl ToJsonResult for serde_json::Value {
fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error> {
Ok(Box::new(self.clone()))
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl ToJsonBytes for serde_json::Value {
fn to_json_bytes(&self) -> Result<Vec<u8>, Error> {
Ok(serde_json::to_vec(self)?)
}
}

View File

@@ -5,10 +5,11 @@ use crate::merger::{Mergeable, Merger};
use crate::merger_cev::ChannelEventsMerger;
use crate::testgen::make_some_boxed_d0_f32;
use crate::Error;
use crate::{binned_collected, runfut, ChannelEvents, Empty, Events, IsoDateTime};
use crate::{binned_collected, runfut, ChannelEvents, Events, IsoDateTime};
use chrono::{TimeZone, Utc};
use futures_util::{stream, StreamExt};
use items::{sitem_data, RangeCompletableItem, Sitemty, StreamItem};
use items_0::Empty;
use netpod::log::*;
use netpod::timeunits::*;
use netpod::{AggKind, BinnedRange, NanoRange, ScalarType, Shape};

View File

@@ -1,5 +1,6 @@
use crate::eventsdim0::EventsDim0;
use crate::{Empty, Events};
use crate::Events;
use items_0::Empty;
#[allow(unused)]
fn xorshift32(state: u32) -> u32 {

View File

@@ -30,6 +30,7 @@ netpod = { path = "../netpod" }
disk = { path = "../disk" }
#parse = { path = "../parse" }
items = { path = "../items" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }
dbconn = { path = "../dbconn" }
scyllaconn = { path = "../scyllaconn" }

View File

@@ -110,7 +110,7 @@ async fn events_conn_handler_inner_try(
let mut p1: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>> =
if evq.channel().backend() == "test-inmem" {
warn!("TEST BACKEND DATA");
use items_2::Empty;
use items_0::Empty;
use netpod::timeunits::MS;
let node_count = node_config.node_config.cluster.nodes.len();
let node_ix = node_config.ix;

View File

@@ -24,4 +24,5 @@ scylla = "0.5"
tokio-postgres = { version = "0.7.7", features = ["with-chrono-0_4", "with-serde_json-1"] }
err = { path = "../err" }
netpod = { path = "../netpod" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }

View File

@@ -2,9 +2,10 @@ use crate::errconv::ErrConv;
use crate::events::EventsStreamScylla;
use err::Error;
use futures_util::{Future, Stream, StreamExt};
use items_0::TimeBinned;
use items_2::binsdim0::BinsDim0;
use items_2::channelevents::ChannelEvents;
use items_2::{empty_binned_dyn, empty_events_dyn, TimeBinned};
use items_2::{empty_binned_dyn, empty_events_dyn};
use netpod::log::*;
use netpod::query::{CacheUsage, PlainEventsQuery, RawEventsQuery};
use netpod::timeunits::*;

View File

@@ -1,9 +1,9 @@
use crate::errconv::ErrConv;
use err::Error;
use futures_util::{Future, FutureExt, Stream, StreamExt};
use items_0::{Empty, Events, WithLen};
use items_2::channelevents::{ChannelEvents, ConnStatus, ConnStatusEvent};
use items_2::eventsdim0::EventsDim0;
use items_2::{Empty, Events, WithLen};
use netpod::log::*;
use netpod::query::{ChannelStateEventsQuery, PlainEventsQuery};
use netpod::timeunits::*;

View File

@@ -20,6 +20,7 @@ chrono = { version = "0.4.19", features = ["serde"] }
err = { path = "../err" }
netpod = { path = "../netpod" }
items = { path = "../items" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }
parse = { path = "../parse" }
bitshuffle = { path = "../bitshuffle" }

View File

@@ -1,7 +1,7 @@
use err::Error;
use futures_util::{Stream, StreamExt};
use items::{RangeCompletableItem, Sitemty, StreamItem};
use items_2::collect::{Collectable, Collector};
use items_0::collect_c::{Collectable, Collector};
use netpod::log::*;
use std::fmt;
use std::time::{Duration, Instant};

View File

@@ -296,8 +296,8 @@ where
#[cfg(test)]
mod test {
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use items_2::Empty;
#[test]
fn merge_channel_events() {

View File

@@ -6,9 +6,9 @@ mod timebin;
use err::Error;
use futures_util::{stream, Stream};
use items::{sitem_data, Sitemty};
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use items_2::eventsdim0::EventsDim0;
use items_2::Empty;
use netpod::timeunits::SEC;
use std::pin::Pin;