daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on events_plain_json_00

Browse Source
This commit is contained in:
Dominik Werder
2022-11-22 11:53:25 +01:00
parent 7cdf5975b9
commit 06e21bc21f
47 changed files with 1133 additions and 687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
items_2/src/binsdim0.rs
View File

@@ -229,6 +229,14 @@ pub struct BinsDim0CollectedResult<NTY> {
finished_at: Option<IsoDateTime>,
}
impl<NTY: ScalarOps> crate::AsAnyRef for BinsDim0CollectedResult<NTY> {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl<NTY: ScalarOps> crate::collect::Collected for BinsDim0CollectedResult<NTY> {}
impl<NTY> BinsDim0CollectedResult<NTY> {
pub fn ts_anchor_sec(&self) -> u64 {
self.ts_anchor_sec

488
items_2/src/channelevents.rs Normal file
View File

@@ -0,0 +1,488 @@
use std::any::Any;
use std::fmt;
use crate::merger_cev::MergeableCev;
use crate::streams::Collectable;
use crate::streams::Collector;
use crate::{merger, Events};
use items::FrameType;
use items::FrameTypeInnerStatic;
use netpod::log::*;
use serde::{Deserialize, Serialize};
// TODO maybe rename to ChannelStatus?
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum ConnStatus {
Connect,
Disconnect,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ConnStatusEvent {
pub ts: u64,
pub status: ConnStatus,
}
impl ConnStatusEvent {
pub fn new(ts: u64, status: ConnStatus) -> Self {
Self { ts, status }
}
}
/// Events on a channel consist not only of e.g. timestamped values, but can be also
/// connection status changes.
#[derive(Debug)]
pub enum ChannelEvents {
Events(Box<dyn Events>),
Status(ConnStatusEvent),
}
impl FrameTypeInnerStatic for ChannelEvents {
const FRAME_TYPE_ID: u32 = items::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
}
impl FrameType for ChannelEvents {
fn frame_type_id(&self) -> u32 {
// TODO SubFrId missing, but get rid of the frame type concept anyhow.
<Self as FrameTypeInnerStatic>::FRAME_TYPE_ID
}
}
impl Clone for ChannelEvents {
fn clone(&self) -> Self {
match self {
Self::Events(arg0) => Self::Events(arg0.clone_dyn()),
Self::Status(arg0) => Self::Status(arg0.clone()),
}
}
}
mod serde_channel_events {
use super::{ChannelEvents, Events};
use crate::eventsdim0::EventsDim0;
use serde::de::{self, EnumAccess, VariantAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
impl Serialize for ChannelEvents {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let name = "ChannelEvents";
match self {
ChannelEvents::Events(obj) => {
use serde::ser::SerializeTupleVariant;
let mut ser = serializer.serialize_tuple_variant(name, 0, "Events", 3)?;
ser.serialize_field(obj.serde_id())?;
ser.serialize_field(&obj.nty_id())?;
ser.serialize_field(obj)?;
ser.end()
}
ChannelEvents::Status(val) => serializer.serialize_newtype_variant(name, 1, "Status", val),
}
}
}
struct EventsBoxVisitor;
impl<'de> Visitor<'de> for EventsBoxVisitor {
type Value = Box<dyn Events>;
fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "Events object")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: de::SeqAccess<'de>,
{
use items::SubFrId;
let e0: &str = seq.next_element()?.ok_or(de::Error::missing_field("ty .0"))?;
let e1: u32 = seq.next_element()?.ok_or(de::Error::missing_field("nty .1"))?;
if e0 == EventsDim0::<u8>::serde_id() {
match e1 {
i32::SUB => {
let obj: EventsDim0<i32> = seq.next_element()?.ok_or(de::Error::missing_field("obj .2"))?;
Ok(Box::new(obj))
}
f32::SUB => {
let obj: EventsDim0<f32> = seq.next_element()?.ok_or(de::Error::missing_field("obj .2"))?;
Ok(Box::new(obj))
}
_ => Err(de::Error::custom(&format!("unknown nty {e1}"))),
}
} else {
Err(de::Error::custom(&format!("unknown ty {e0}")))
}
}
}
pub struct ChannelEventsVisitor;
impl ChannelEventsVisitor {
fn name() -> &'static str {
"ChannelEvents"
}
fn allowed_variants() -> &'static [&'static str] {
&["Events", "Status", "RangeComplete"]
}
}
impl<'de> Visitor<'de> for ChannelEventsVisitor {
type Value = ChannelEvents;
fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "ChannelEvents")
}
fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
where
A: EnumAccess<'de>,
{
let (id, var) = data.variant()?;
match id {
"Events" => {
let c = var.tuple_variant(3, EventsBoxVisitor)?;
Ok(Self::Value::Events(c))
}
_ => return Err(de::Error::unknown_variant(id, Self::allowed_variants())),
}
}
}
impl<'de> Deserialize<'de> for ChannelEvents {
fn deserialize<D>(de: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
de.deserialize_enum(
ChannelEventsVisitor::name(),
ChannelEventsVisitor::allowed_variants(),
ChannelEventsVisitor,
)
}
}
}
#[cfg(test)]
mod test_channel_events_serde {
use super::ChannelEvents;
use crate::eventsdim0::EventsDim0;
use crate::Empty;
#[test]
fn channel_events() {
let mut evs = EventsDim0::empty();
evs.push(8, 2, 3.0f32);
evs.push(12, 3, 3.2f32);
let item = ChannelEvents::Events(Box::new(evs));
let s = serde_json::to_string_pretty(&item).unwrap();
eprintln!("{s}");
let w: ChannelEvents = serde_json::from_str(&s).unwrap();
eprintln!("{w:?}");
}
}
impl PartialEq for ChannelEvents {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Events(l0), Self::Events(r0)) => l0 == r0,
(Self::Status(l0), Self::Status(r0)) => l0 == r0,
_ => core::mem::discriminant(self) == core::mem::discriminant(other),
}
}
}
impl MergeableCev for ChannelEvents {
fn ts_min(&self) -> Option<u64> {
use ChannelEvents::*;
match self {
Events(k) => k.ts_min(),
Status(k) => Some(k.ts),
}
}
fn ts_max(&self) -> Option<u64> {
error!("TODO impl MergableEvents for ChannelEvents");
err::todoval()
}
}
impl crate::merger::Mergeable for ChannelEvents {
fn len(&self) -> usize {
match self {
ChannelEvents::Events(k) => k.len(),
ChannelEvents::Status(_) => 1,
}
}
fn ts_min(&self) -> Option<u64> {
match self {
ChannelEvents::Events(k) => k.ts_min(),
ChannelEvents::Status(k) => Some(k.ts),
}
}
fn ts_max(&self) -> Option<u64> {
match self {
ChannelEvents::Events(k) => k.ts_max(),
ChannelEvents::Status(k) => Some(k.ts),
}
}
fn is_compatible_target(&self, tgt: &Self) -> bool {
use ChannelEvents::*;
match self {
Events(_) => {
// TODO better to delegate this to inner type?
if let Events(_) = tgt {
true
} else {
false
}
}
Status(_) => {
// TODO better to delegate this to inner type?
if let Status(_) = tgt {
true
} else {
false
}
}
}
}
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
match self {
ChannelEvents::Events(k) => ChannelEvents::Events(k.move_into_fresh(ts_end)),
ChannelEvents::Status(k) => ChannelEvents::Status(k.clone()),
}
}
fn move_into_existing(&mut self, tgt: &mut Self, ts_end: u64) -> Result<(), merger::MergeError> {
match self {
ChannelEvents::Events(k) => match tgt {
ChannelEvents::Events(tgt) => k.move_into_existing(tgt, ts_end),
ChannelEvents::Status(_) => Err(merger::MergeError::NotCompatible),
},
ChannelEvents::Status(_) => match tgt {
ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible),
ChannelEvents::Status(_) => Err(merger::MergeError::Full),
},
}
}
}
impl Collectable for ChannelEvents {
fn new_collector(&self) -> Box<dyn Collector> {
match self {
ChannelEvents::Events(_item) => todo!(),
ChannelEvents::Status(_) => todo!(),
}
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
pub struct ChannelEventsTimeBinner {
// TODO `ConnStatus` contains all the changes that can happen to a connection, but
// here we would rather require a simplified current state for binning purpose.
edges: Vec<u64>,
do_time_weight: bool,
conn_state: ConnStatus,
binner: Option<Box<dyn crate::TimeBinner>>,
}
impl fmt::Debug for ChannelEventsTimeBinner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ChannelEventsTimeBinner")
.field("conn_state", &self.conn_state)
.finish()
}
}
impl crate::timebin::TimeBinner for ChannelEventsTimeBinner {
type Input = ChannelEvents;
type Output = Box<dyn crate::TimeBinned>;
fn ingest(&mut self, item: &mut Self::Input) {
match item {
ChannelEvents::Events(item) => {
if self.binner.is_none() {
let binner = item.time_binner_new(self.edges.clone(), self.do_time_weight);
self.binner = Some(binner);
}
match self.binner.as_mut() {
Some(binner) => binner.ingest(item.as_time_binnable()),
None => {
error!("ingest without active binner item {item:?}");
()
}
}
}
ChannelEvents::Status(item) => {
warn!("TODO consider channel status in time binning {item:?}");
}
}
}
fn set_range_complete(&mut self) {
match self.binner.as_mut() {
Some(binner) => binner.set_range_complete(),
None => (),
}
}
fn bins_ready_count(&self) -> usize {
match &self.binner {
Some(binner) => binner.bins_ready_count(),
None => 0,
}
}
fn bins_ready(&mut self) -> Option<Self::Output> {
match self.binner.as_mut() {
Some(binner) => binner.bins_ready(),
None => None,
}
}
fn push_in_progress(&mut self, push_empty: bool) {
match self.binner.as_mut() {
Some(binner) => binner.push_in_progress(push_empty),
None => (),
}
}
fn cycle(&mut self) {
match self.binner.as_mut() {
Some(binner) => binner.cycle(),
None => (),
}
}
}
impl crate::timebin::TimeBinnable for ChannelEvents {
type TimeBinner = ChannelEventsTimeBinner;
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Self::TimeBinner {
let (binner, status) = match self {
ChannelEvents::Events(_events) => (None, ConnStatus::Connect),
ChannelEvents::Status(status) => (None, status.status.clone()),
};
ChannelEventsTimeBinner {
edges,
do_time_weight,
conn_state: status,
binner,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ChannelEventsCollectorOutput {}
impl crate::AsAnyRef for ChannelEventsCollectorOutput {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl crate::ToJsonResult for ChannelEventsCollectorOutput {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, err::Error> {
todo!()
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl crate::collect::Collected for ChannelEventsCollectorOutput {}
#[derive(Debug)]
pub struct ChannelEventsCollector {
coll: Option<Box<dyn crate::collect::CollectorDyn>>,
range_complete: bool,
timed_out: bool,
}
impl ChannelEventsCollector {
pub fn new() -> Self {
Self {
coll: None,
range_complete: false,
timed_out: false,
}
}
}
impl crate::collect::Collector for ChannelEventsCollector {
type Input = ChannelEvents;
type Output = Box<dyn crate::collect::Collected>;
fn len(&self) -> usize {
match &self.coll {
Some(coll) => coll.len(),
None => 0,
}
}
fn ingest(&mut self, item: &mut Self::Input) {
match item {
ChannelEvents::Events(item) => {
if self.coll.is_none() {
let coll = item.as_ref().as_collectable_with_default_ref().new_collector();
self.coll = Some(coll);
}
let coll = self.coll.as_mut().unwrap();
coll.ingest(item.as_collectable_with_default_mut());
}
ChannelEvents::Status(_) => {
// TODO decide on output format to collect also the connection status events
}
}
}
fn set_range_complete(&mut self) {
self.range_complete = true;
}
fn set_timed_out(&mut self) {
self.timed_out = true;
}
fn result(&mut self) -> Result<Self::Output, crate::Error> {
match self.coll.as_mut() {
Some(coll) => {
if self.range_complete {
coll.set_range_complete();
}
if self.timed_out {
coll.set_timed_out();
}
let res = coll.result()?;
//error!("fix output of ChannelEventsCollector [03ce6bc5a]");
//err::todo();
//let output = ChannelEventsCollectorOutput {};
Ok(res)
}
None => {
error!("nothing collected [caa8d2565]");
todo!()
}
}
}
}
impl crate::collect::Collectable for ChannelEvents {
type Collector = ChannelEventsCollector;
fn new_collector(&self) -> Self::Collector {
ChannelEventsCollector::new()
}
}

49
items_2/src/collect.rs
View File

@@ -1,9 +1,12 @@
use crate::AsAnyRef;
use crate::Error;
use std::any::Any;
use std::fmt;
pub trait Collector: fmt::Debug {
pub trait Collector: fmt::Debug + Send {
// TODO should require here Collectable?
type Input;
type Output;
type Output: Collected;
fn len(&self) -> usize;
@@ -21,3 +24,45 @@ pub trait Collectable: fmt::Debug {
fn new_collector(&self) -> Self::Collector;
}
pub trait Collected: fmt::Debug + crate::streams::ToJsonResult + AsAnyRef + Send {}
erased_serde::serialize_trait_object!(Collected);
impl AsAnyRef for Box<dyn Collected> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}
impl crate::streams::ToJsonResult for Box<dyn Collected> {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, err::Error> {
self.as_ref().to_json_result()
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl Collected for Box<dyn Collected> {}
#[derive(Debug)]
pub struct CollectorDynDefault {}
pub trait CollectorDyn: fmt::Debug + Send {
fn len(&self) -> usize;
fn ingest(&mut self, item: &mut dyn CollectableWithDefault);
fn set_range_complete(&mut self);
fn set_timed_out(&mut self);
fn result(&mut self) -> Result<Box<dyn Collected>, Error>;
}
pub trait CollectableWithDefault {
fn new_collector(&self) -> Box<dyn CollectorDyn>;
fn as_any_mut(&mut self) -> &mut dyn Any;
}

104
items_2/src/eventsdim0.rs
View File

@@ -164,6 +164,20 @@ pub struct EventsDim0CollectorOutput<NTY> {
timed_out: bool,
}
impl<NTY: ScalarOps> EventsDim0CollectorOutput<NTY> {
pub fn len(&self) -> usize {
self.values.len()
}
}
impl<NTY: ScalarOps> crate::AsAnyRef for EventsDim0CollectorOutput<NTY> {
fn as_any_ref(&self) -> &dyn Any {
self
}
}
impl<NTY: ScalarOps> crate::collect::Collected for EventsDim0CollectorOutput<NTY> {}
impl<NTY: ScalarOps> ToJsonResult for EventsDim0CollectorOutput<NTY> {
fn to_json_result(&self) -> Result<Box<dyn crate::streams::ToJsonBytes>, Error> {
let k = serde_json::to_value(self)?;
@@ -222,7 +236,6 @@ impl<NTY: ScalarOps> CollectableType for EventsDim0<NTY> {
impl<NTY: ScalarOps> crate::collect::Collector for EventsDim0Collector<NTY> {
type Input = EventsDim0<NTY>;
// TODO the output probably needs to be different to accommodate also range-complete, continue-at, etc
type Output = EventsDim0CollectorOutput<NTY>;
fn len(&self) -> usize {
@@ -576,6 +589,14 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
self
}
fn as_collectable_with_default_ref(&self) -> &dyn crate::collect::CollectableWithDefault {
self
}
fn as_collectable_with_default_mut(&mut self) -> &mut dyn crate::collect::CollectableWithDefault {
self
}
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events> {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
@@ -598,7 +619,7 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrealted traits. Simplify.
if let Some(tgt) = tgt.as_any_mut().downcast_mut::<Self>() {
if let Some(tgt) = crate::streams::Collectable::as_any_mut(tgt.as_mut()).downcast_mut::<Self>() {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
// TODO make it harder to forget new members when the struct may get modified in the future
@@ -837,10 +858,83 @@ impl<NTY: ScalarOps> TimeBinner for EventsDim0TimeBinner<NTY> {
}
}
impl<NTY: ScalarOps> crate::collect::Collectable for EventsDim0<NTY> {
type Collector;
#[derive(Debug)]
pub struct EventsDim0CollectorDyn {}
fn new_collector(&self) -> Self::Collector {
impl EventsDim0CollectorDyn {
pub fn new() -> Self {
Self {}
}
}
impl crate::collect::CollectorDyn for EventsDim0CollectorDyn {
fn len(&self) -> usize {
todo!()
}
fn ingest(&mut self, _item: &mut dyn crate::collect::CollectableWithDefault) {
// TODO remove this struct?
todo!()
}
fn set_range_complete(&mut self) {
todo!()
}
fn set_timed_out(&mut self) {
todo!()
}
fn result(&mut self) -> Result<Box<dyn crate::collect::Collected>, crate::Error> {
todo!()
}
}
impl<NTY: ScalarOps> crate::collect::CollectorDyn for EventsDim0Collector<NTY> {
fn len(&self) -> usize {
WithLen::len(self)
}
fn ingest(&mut self, item: &mut dyn crate::collect::CollectableWithDefault) {
let x = item.as_any_mut();
if let Some(item) = x.downcast_mut::<EventsDim0<NTY>>() {
CollectorType::ingest(self, item)
} else {
// TODO need possibility to return error
()
}
}
fn set_range_complete(&mut self) {
CollectorType::set_range_complete(self);
}
fn set_timed_out(&mut self) {
CollectorType::set_timed_out(self);
}
fn result(&mut self) -> Result<Box<dyn crate::collect::Collected>, crate::Error> {
CollectorType::result(self)
.map(|x| Box::new(x) as _)
.map_err(|e| e.into())
}
}
impl<NTY: ScalarOps> crate::collect::CollectableWithDefault for EventsDim0<NTY> {
fn new_collector(&self) -> Box<dyn crate::collect::CollectorDyn> {
let coll = EventsDim0Collector::<NTY>::new();
Box::new(coll)
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
impl<NTY: ScalarOps> crate::collect::Collectable for EventsDim0<NTY> {
type Collector = EventsDim0Collector<NTY>;
fn new_collector(&self) -> Self::Collector {
EventsDim0Collector::new()
}
}

492
items_2/src/items_2.rs
View File

@@ -1,4 +1,5 @@
pub mod binsdim0;
pub mod channelevents;
pub mod collect;
pub mod eventsdim0;
pub mod merger;
@@ -9,17 +10,19 @@ pub mod test;
pub mod testgen;
pub mod timebin;
use crate as items_2;
use crate::streams::Collectable;
use crate::streams::Collector;
use crate::streams::ToJsonResult;
use channelevents::ChannelEvents;
use chrono::{DateTime, TimeZone, Utc};
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use items::FrameTypeInnerStatic;
use items::RangeCompletableItem;
use items::Sitemty;
use items::StreamItem;
use items::SubFrId;
use merger_cev::MergeableCev;
use netpod::log::*;
use netpod::timeunits::*;
use netpod::{AggKind, NanoRange, ScalarType, Shape};
@@ -30,8 +33,6 @@ use std::fmt;
use std::pin::Pin;
use std::time::Duration;
use std::time::Instant;
use streams::Collectable;
use streams::ToJsonResult;
pub fn bool_is_false(x: &bool) -> bool {
*x == false
@@ -262,6 +263,20 @@ pub trait AppendEmptyBin {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64);
}
pub trait AsAnyRef {
fn as_any_ref(&self) -> &dyn Any;
}
pub trait AsAnyMut {
fn as_any_mut(&mut self) -> &mut dyn Any;
}
/*impl AsAnyRef for Box<dyn AsAnyRef> {
fn as_any_ref(&self) -> &dyn Any {
self.as_ref().as_any_ref()
}
}*/
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct IsoDateTime(DateTime<Utc>);
@@ -313,11 +328,21 @@ pub trait TimeBinnable: fmt::Debug + WithLen + RangeOverlapInfo + Any + Send {
// TODO can I remove the Any bound?
/// Container of some form of events, for use as trait object.
pub trait Events: fmt::Debug + Any + Collectable + TimeBinnable + Send + erased_serde::Serialize {
pub trait Events:
fmt::Debug
+ Any
+ Collectable
+ items_2::collect::CollectableWithDefault
+ TimeBinnable
+ Send
+ erased_serde::Serialize
{
fn as_time_binnable(&self) -> &dyn TimeBinnable;
fn verify(&self) -> bool;
fn output_info(&self);
fn as_collectable_mut(&mut self) -> &mut dyn Collectable;
fn as_collectable_with_default_ref(&self) -> &dyn crate::collect::CollectableWithDefault;
fn as_collectable_with_default_mut(&mut self) -> &mut dyn crate::collect::CollectableWithDefault;
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events>;
@@ -366,74 +391,6 @@ impl PartialEq for Box<dyn Events> {
}
}
// TODO remove
struct EventsCollector2 {}
impl WithLen for EventsCollector2 {
fn len(&self) -> usize {
todo!()
}
}
// TODO remove
impl Collector for EventsCollector2 {
fn ingest(&mut self, _src: &mut dyn Collectable) {
todo!()
}
fn set_range_complete(&mut self) {
todo!()
}
fn set_timed_out(&mut self) {
todo!()
}
fn result(&mut self) -> Result<Box<dyn ToJsonResult>, err::Error> {
todo!()
}
}
// TODO remove
impl Collectable for Box<dyn Events> {
fn new_collector(&self) -> Box<dyn Collector> {
Box::new(EventsCollector2 {})
}
fn as_any_mut(&mut self) -> &mut dyn Any {
todo!()
}
}
/*impl crate::streams::CollectorType for EventsCollector {
type Input = dyn Events;
type Output = Box<dyn Events>;
fn ingest(&mut self, src: &mut Self::Input) {
todo!()
}
fn set_range_complete(&mut self) {
todo!()
}
fn set_timed_out(&mut self) {
todo!()
}
fn result(&mut self) -> Result<Self::Output, err::Error> {
todo!()
}
}*/
/*impl crate::streams::CollectableType for dyn Events {
type Collector = EventsCollector;
fn new_collector() -> Self::Collector {
todo!()
}
}*/
/// Data in time-binned form.
pub trait TimeBinned: Any + TimeBinnable {
fn as_time_binnable_dyn(&self) -> &dyn TimeBinnable;
@@ -593,409 +550,50 @@ pub fn empty_binned_dyn(scalar_type: &ScalarType, shape: &Shape, agg_kind: &AggK
}
}
// TODO maybe rename to ChannelStatus?
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum ConnStatus {
Connect,
Disconnect,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ConnStatusEvent {
pub ts: u64,
pub status: ConnStatus,
}
impl ConnStatusEvent {
pub fn new(ts: u64, status: ConnStatus) -> Self {
Self { ts, status }
}
}
/// Events on a channel consist not only of e.g. timestamped values, but can be also
/// connection status changes.
#[derive(Debug)]
pub enum ChannelEvents {
Events(Box<dyn Events>),
Status(ConnStatusEvent),
}
impl FrameTypeInnerStatic for ChannelEvents {
const FRAME_TYPE_ID: u32 = items::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
}
impl Clone for ChannelEvents {
fn clone(&self) -> Self {
match self {
Self::Events(arg0) => Self::Events(arg0.clone_dyn()),
Self::Status(arg0) => Self::Status(arg0.clone()),
}
}
}
mod serde_channel_events {
use super::{ChannelEvents, Events};
use crate::eventsdim0::EventsDim0;
use serde::de::{self, EnumAccess, VariantAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
impl Serialize for ChannelEvents {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let name = "ChannelEvents";
match self {
ChannelEvents::Events(obj) => {
use serde::ser::SerializeTupleVariant;
let mut ser = serializer.serialize_tuple_variant(name, 0, "Events", 3)?;
ser.serialize_field(obj.serde_id())?;
ser.serialize_field(&obj.nty_id())?;
ser.serialize_field(obj)?;
ser.end()
}
ChannelEvents::Status(val) => serializer.serialize_newtype_variant(name, 1, "Status", val),
}
}
}
struct EventsBoxVisitor;
impl<'de> Visitor<'de> for EventsBoxVisitor {
type Value = Box<dyn Events>;
fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "Events object")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: de::SeqAccess<'de>,
{
use items::SubFrId;
let e0: &str = seq.next_element()?.ok_or(de::Error::missing_field("ty .0"))?;
let e1: u32 = seq.next_element()?.ok_or(de::Error::missing_field("nty .1"))?;
if e0 == EventsDim0::<u8>::serde_id() {
match e1 {
f32::SUB => {
let obj: EventsDim0<f32> = seq.next_element()?.ok_or(de::Error::missing_field("obj .2"))?;
Ok(Box::new(obj))
}
_ => Err(de::Error::custom(&format!("unknown nty {e1}"))),
}
} else {
Err(de::Error::custom(&format!("unknown ty {e0}")))
}
}
}
pub struct ChannelEventsVisitor;
impl ChannelEventsVisitor {
fn name() -> &'static str {
"ChannelEvents"
}
fn allowed_variants() -> &'static [&'static str] {
&["Events", "Status", "RangeComplete"]
}
}
impl<'de> Visitor<'de> for ChannelEventsVisitor {
type Value = ChannelEvents;
fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "ChannelEvents")
}
fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
where
A: EnumAccess<'de>,
{
let (id, var) = data.variant()?;
match id {
"Events" => {
let c = var.tuple_variant(3, EventsBoxVisitor)?;
Ok(Self::Value::Events(c))
}
_ => return Err(de::Error::unknown_variant(id, Self::allowed_variants())),
}
}
}
impl<'de> Deserialize<'de> for ChannelEvents {
fn deserialize<D>(de: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
de.deserialize_enum(
ChannelEventsVisitor::name(),
ChannelEventsVisitor::allowed_variants(),
ChannelEventsVisitor,
)
}
}
}
#[cfg(test)]
mod test_channel_events_serde {
use super::ChannelEvents;
use crate::eventsdim0::EventsDim0;
use crate::Empty;
#[test]
fn channel_events() {
let mut evs = EventsDim0::empty();
evs.push(8, 2, 3.0f32);
evs.push(12, 3, 3.2f32);
let item = ChannelEvents::Events(Box::new(evs));
let s = serde_json::to_string_pretty(&item).unwrap();
eprintln!("{s}");
let w: ChannelEvents = serde_json::from_str(&s).unwrap();
eprintln!("{w:?}");
}
}
impl PartialEq for ChannelEvents {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Events(l0), Self::Events(r0)) => l0 == r0,
(Self::Status(l0), Self::Status(r0)) => l0 == r0,
_ => core::mem::discriminant(self) == core::mem::discriminant(other),
}
}
}
impl MergeableCev for ChannelEvents {
fn ts_min(&self) -> Option<u64> {
use ChannelEvents::*;
match self {
Events(k) => k.ts_min(),
Status(k) => Some(k.ts),
}
}
fn ts_max(&self) -> Option<u64> {
error!("TODO impl MergableEvents for ChannelEvents");
err::todoval()
}
}
impl crate::merger::Mergeable for ChannelEvents {
fn len(&self) -> usize {
match self {
ChannelEvents::Events(k) => k.len(),
ChannelEvents::Status(_) => 1,
}
}
fn ts_min(&self) -> Option<u64> {
match self {
ChannelEvents::Events(k) => k.ts_min(),
ChannelEvents::Status(k) => Some(k.ts),
}
}
fn ts_max(&self) -> Option<u64> {
match self {
ChannelEvents::Events(k) => k.ts_max(),
ChannelEvents::Status(k) => Some(k.ts),
}
}
fn is_compatible_target(&self, tgt: &Self) -> bool {
use ChannelEvents::*;
match self {
Events(_) => {
// TODO better to delegate this to inner type?
if let Events(_) = tgt {
true
} else {
false
}
}
Status(_) => {
// TODO better to delegate this to inner type?
if let Status(_) = tgt {
true
} else {
false
}
}
}
}
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
match self {
ChannelEvents::Events(k) => ChannelEvents::Events(k.move_into_fresh(ts_end)),
ChannelEvents::Status(k) => ChannelEvents::Status(k.clone()),
}
}
fn move_into_existing(&mut self, tgt: &mut Self, ts_end: u64) -> Result<(), merger::MergeError> {
match self {
ChannelEvents::Events(k) => match tgt {
ChannelEvents::Events(tgt) => k.move_into_existing(tgt, ts_end),
ChannelEvents::Status(_) => Err(merger::MergeError::NotCompatible),
},
ChannelEvents::Status(_) => match tgt {
ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible),
ChannelEvents::Status(_) => Err(merger::MergeError::Full),
},
}
}
}
impl Collectable for ChannelEvents {
fn new_collector(&self) -> Box<dyn Collector> {
match self {
ChannelEvents::Events(_item) => todo!(),
ChannelEvents::Status(_) => todo!(),
}
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
pub struct ChannelEventsTimeBinner {
// TODO `ConnStatus` contains all the changes that can happen to a connection, but
// here we would rather require a simplified current state for binning purpose.
edges: Vec<u64>,
do_time_weight: bool,
conn_state: ConnStatus,
binner: Option<Box<dyn crate::TimeBinner>>,
}
impl fmt::Debug for ChannelEventsTimeBinner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ChannelEventsTimeBinner")
.field("conn_state", &self.conn_state)
.finish()
}
}
impl crate::timebin::TimeBinner for ChannelEventsTimeBinner {
type Input = ChannelEvents;
type Output = Box<dyn crate::TimeBinned>;
fn ingest(&mut self, item: &mut Self::Input) {
match item {
ChannelEvents::Events(item) => {
if self.binner.is_none() {
let binner = item.time_binner_new(self.edges.clone(), self.do_time_weight);
self.binner = Some(binner);
}
match self.binner.as_mut() {
Some(binner) => binner.ingest(item.as_time_binnable()),
None => {
error!("ingest without active binner item {item:?}");
()
}
}
}
ChannelEvents::Status(item) => {
warn!("TODO consider channel status in time binning {item:?}");
}
}
}
fn set_range_complete(&mut self) {
match self.binner.as_mut() {
Some(binner) => binner.set_range_complete(),
None => (),
}
}
fn bins_ready_count(&self) -> usize {
match &self.binner {
Some(binner) => binner.bins_ready_count(),
None => 0,
}
}
fn bins_ready(&mut self) -> Option<Self::Output> {
match self.binner.as_mut() {
Some(binner) => binner.bins_ready(),
None => None,
}
}
fn push_in_progress(&mut self, push_empty: bool) {
match self.binner.as_mut() {
Some(binner) => binner.push_in_progress(push_empty),
None => (),
}
}
fn cycle(&mut self) {
match self.binner.as_mut() {
Some(binner) => binner.cycle(),
None => (),
}
}
}
impl crate::timebin::TimeBinnable for ChannelEvents {
type TimeBinner = ChannelEventsTimeBinner;
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Self::TimeBinner {
let (binner, status) = match self {
ChannelEvents::Events(_events) => (None, ConnStatus::Connect),
ChannelEvents::Status(status) => (None, status.status.clone()),
};
ChannelEventsTimeBinner {
edges,
do_time_weight,
conn_state: status,
binner,
}
}
}
#[derive(Debug)]
pub struct EventsCollector {
coll: Option<Box<()>>,
coll: Box<dyn items_2::collect::CollectorDyn>,
}
impl EventsCollector {
pub fn new() -> Self {
Self { coll: Box::new(()) }
pub fn new(coll: Box<dyn items_2::collect::CollectorDyn>) -> Self {
Self { coll }
}
}
impl crate::collect::Collector for EventsCollector {
impl items_2::collect::Collector for EventsCollector {
type Input = Box<dyn Events>;
type Output = Box<dyn Events>;
// TODO this Output trait does not differentiate between e.g. collected events, collected bins, different aggs, etc...
type Output = Box<dyn items_2::collect::Collected>;
fn len(&self) -> usize {
todo!()
self.coll.len()
}
fn ingest(&mut self, item: &mut Self::Input) {
todo!()
self.coll.ingest(item.as_collectable_with_default_mut());
}
fn set_range_complete(&mut self) {
todo!()
self.coll.set_range_complete()
}
fn set_timed_out(&mut self) {
todo!()
self.coll.set_timed_out()
}
fn result(&mut self) -> Result<Self::Output, Error> {
todo!()
self.coll.result()
}
}
impl crate::collect::Collectable for Box<dyn Events> {
impl items_2::collect::Collectable for Box<dyn Events> {
type Collector = EventsCollector;
fn new_collector(&self) -> Self::Collector {
Collectable::new_collector(self)
let coll = items_2::collect::CollectableWithDefault::new_collector(self.as_ref());
EventsCollector::new(coll)
}
}

2
items_2/src/streams.rs
View File

@@ -6,7 +6,7 @@ use std::fmt;
pub trait CollectorType: Send + Unpin + WithLen {
type Input: Collectable;
type Output: ToJsonResult + Serialize;
type Output: crate::collect::Collected + ToJsonResult + Serialize;
fn ingest(&mut self, src: &mut Self::Input);
fn set_range_complete(&mut self);

3
items_2/src/test.rs
View File

@@ -1,10 +1,11 @@
use crate::binsdim0::BinsDim0CollectedResult;
use crate::channelevents::{ConnStatus, ConnStatusEvent};
use crate::eventsdim0::EventsDim0;
use crate::merger::{Mergeable, Merger};
use crate::merger_cev::ChannelEventsMerger;
use crate::testgen::make_some_boxed_d0_f32;
use crate::Error;
use crate::{binned_collected, runfut, ChannelEvents, Empty, Events, IsoDateTime};
use crate::{ConnStatus, ConnStatusEvent, Error};
use chrono::{TimeZone, Utc};
use futures_util::{stream, StreamExt};
use items::{sitem_data, RangeCompletableItem, Sitemty, StreamItem};