This commit is contained in:
Dominik Werder
2023-02-10 12:37:39 +01:00
parent 4d9a33b77f
commit b453d61250
10 changed files with 120 additions and 132 deletions

View File

@@ -151,9 +151,8 @@ pub trait Events:
fn as_collectable_with_default_mut(&mut self) -> &mut dyn CollectableWithDefault; fn as_collectable_with_default_mut(&mut self) -> &mut dyn CollectableWithDefault;
fn ts_min(&self) -> Option<u64>; fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>; fn ts_max(&self) -> Option<u64>;
// TODO is this used?
fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events>; fn take_new_events_until_ts(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_fresh(&mut self, ts_end: u64) -> Box<dyn Events>;
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()>;
fn new_empty(&self) -> Box<dyn Events>; fn new_empty(&self) -> Box<dyn Events>;
fn drain_into(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), ()>; fn drain_into(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), ()>;
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>; fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>;

View File

@@ -85,7 +85,7 @@ impl ChannelStatusEvent {
#[derive(Debug)] #[derive(Debug)]
pub enum ChannelEvents { pub enum ChannelEvents {
Events(Box<dyn Events>), Events(Box<dyn Events>),
Status(ConnStatusEvent), Status(Option<ConnStatusEvent>),
} }
impl FrameTypeInnerStatic for ChannelEvents { impl FrameTypeInnerStatic for ChannelEvents {
@@ -342,7 +342,7 @@ mod serde_channel_events {
Ok(Self::Value::Events(x.0)) Ok(Self::Value::Events(x.0))
} }
VarId::Status => { VarId::Status => {
let x: ConnStatusEvent = var.newtype_variant()?; let x: Option<ConnStatusEvent> = var.newtype_variant()?;
Ok(Self::Value::Status(x)) Ok(Self::Value::Status(x))
} }
} }
@@ -440,7 +440,7 @@ mod test_channel_events_serde {
datetime: SystemTime::UNIX_EPOCH, datetime: SystemTime::UNIX_EPOCH,
status: crate::channelevents::ConnStatus::Connect, status: crate::channelevents::ConnStatus::Connect,
}; };
let item = ChannelEvents::Status(status); let item = ChannelEvents::Status(Some(status));
let opts = bincode_opts(); let opts = bincode_opts();
let mut out = Vec::new(); let mut out = Vec::new();
let mut ser = bincode::Serializer::new(&mut out, opts); let mut ser = bincode::Serializer::new(&mut out, opts);
@@ -453,7 +453,11 @@ mod test_channel_events_serde {
} else { } else {
panic!() panic!()
}; };
assert_eq!(item.ts, 567); if let Some(item) = item {
assert_eq!(item.ts, 567);
} else {
panic!()
}
} }
} }
@@ -472,7 +476,10 @@ impl MergeableCev for ChannelEvents {
use ChannelEvents::*; use ChannelEvents::*;
match self { match self {
Events(k) => k.ts_min(), Events(k) => k.ts_min(),
Status(k) => Some(k.ts), Status(k) => match k {
Some(k) => Some(k.ts),
None => None,
},
} }
} }
@@ -486,40 +493,29 @@ impl crate::merger::Mergeable for ChannelEvents {
fn len(&self) -> usize { fn len(&self) -> usize {
match self { match self {
ChannelEvents::Events(k) => k.len(), ChannelEvents::Events(k) => k.len(),
ChannelEvents::Status(_) => 1, ChannelEvents::Status(k) => match k {
Some(_) => 1,
None => 0,
},
} }
} }
fn ts_min(&self) -> Option<u64> { fn ts_min(&self) -> Option<u64> {
match self { match self {
ChannelEvents::Events(k) => k.ts_min(), ChannelEvents::Events(k) => k.ts_min(),
ChannelEvents::Status(k) => Some(k.ts), ChannelEvents::Status(k) => match k {
Some(k) => Some(k.ts),
None => None,
},
} }
} }
fn ts_max(&self) -> Option<u64> { fn ts_max(&self) -> Option<u64> {
match self { match self {
ChannelEvents::Events(k) => k.ts_max(), ChannelEvents::Events(k) => k.ts_max(),
ChannelEvents::Status(k) => Some(k.ts), ChannelEvents::Status(k) => match k {
} Some(k) => Some(k.ts),
} None => None,
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
match self {
ChannelEvents::Events(k) => ChannelEvents::Events(k.move_into_fresh(ts_end)),
ChannelEvents::Status(k) => ChannelEvents::Status(k.clone()),
}
}
fn move_into_existing(&mut self, tgt: &mut Self, ts_end: u64) -> Result<(), merger::MergeError> {
match self {
ChannelEvents::Events(k) => match tgt {
ChannelEvents::Events(tgt) => k.move_into_existing(tgt, ts_end),
ChannelEvents::Status(_) => Err(merger::MergeError::NotCompatible),
},
ChannelEvents::Status(_) => match tgt {
ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible),
ChannelEvents::Status(_) => Err(merger::MergeError::Full),
}, },
} }
} }
@@ -539,11 +535,13 @@ impl crate::merger::Mergeable for ChannelEvents {
}, },
ChannelEvents::Status(k) => match dst { ChannelEvents::Status(k) => match dst {
ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible), ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible),
ChannelEvents::Status(j) => { ChannelEvents::Status(j) => match j {
// TODO must have some empty-value for the status container. Some(_) => Err(merger::MergeError::Full),
*j = k.clone(); None => {
Ok(()) *j = k.take();
} Ok(())
}
},
}, },
} }
} }
@@ -552,8 +550,12 @@ impl crate::merger::Mergeable for ChannelEvents {
match self { match self {
ChannelEvents::Events(k) => k.find_lowest_index_gt(ts), ChannelEvents::Events(k) => k.find_lowest_index_gt(ts),
ChannelEvents::Status(k) => { ChannelEvents::Status(k) => {
if k.ts > ts { if let Some(k) = k {
Some(0) if k.ts > ts {
Some(0)
} else {
None
}
} else { } else {
None None
} }
@@ -565,8 +567,12 @@ impl crate::merger::Mergeable for ChannelEvents {
match self { match self {
ChannelEvents::Events(k) => k.find_lowest_index_ge(ts), ChannelEvents::Events(k) => k.find_lowest_index_ge(ts),
ChannelEvents::Status(k) => { ChannelEvents::Status(k) => {
if k.ts >= ts { if let Some(k) = k {
Some(0) if k.ts >= ts {
Some(0)
} else {
None
}
} else { } else {
None None
} }
@@ -578,8 +584,12 @@ impl crate::merger::Mergeable for ChannelEvents {
match self { match self {
ChannelEvents::Events(k) => k.find_highest_index_lt(ts), ChannelEvents::Events(k) => k.find_highest_index_lt(ts),
ChannelEvents::Status(k) => { ChannelEvents::Status(k) => {
if k.ts < ts { if let Some(k) = k {
Some(0) if k.ts < ts {
Some(0)
} else {
None
}
} else { } else {
None None
} }
@@ -686,9 +696,10 @@ impl crate::timebin::TimeBinnable for ChannelEvents {
type TimeBinner = ChannelEventsTimeBinner; type TimeBinner = ChannelEventsTimeBinner;
fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Self::TimeBinner { fn time_binner_new(&self, edges: Vec<u64>, do_time_weight: bool) -> Self::TimeBinner {
// TODO probably wrong?
let (binner, status) = match self { let (binner, status) = match self {
ChannelEvents::Events(_events) => (None, ConnStatus::Connect), ChannelEvents::Events(_events) => (None, ConnStatus::Connect),
ChannelEvents::Status(status) => (None, status.status.clone()), ChannelEvents::Status(_status) => (None, ConnStatus::Connect),
}; };
ChannelEventsTimeBinner { ChannelEventsTimeBinner {
edges, edges,

View File

@@ -778,32 +778,6 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
Box::new(ret) Box::new(ret)
} }
fn move_into_fresh(&mut self, ts_end: u64) -> Box<dyn Events> {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
let tss = self.tss.drain(..n1).collect();
let pulses = self.pulses.drain(..n1).collect();
let values = self.values.drain(..n1).collect();
let ret = Self { tss, pulses, values };
Box::new(ret)
}
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrealted traits. Simplify.
if let Some(tgt) = tgt.as_mut().as_any_mut().downcast_mut::<Self>() {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
// TODO make it harder to forget new members when the struct may get modified in the future
tgt.tss.extend(self.tss.drain(..n1));
tgt.pulses.extend(self.pulses.drain(..n1));
tgt.values.extend(self.values.drain(..n1));
Ok(())
} else {
eprintln!("downcast to EventsDim0 FAILED");
Err(())
}
}
fn new_empty(&self) -> Box<dyn Events> { fn new_empty(&self) -> Box<dyn Events> {
Box::new(Self::empty()) Box::new(Self::empty())
} }

View File

@@ -730,32 +730,6 @@ impl<NTY: ScalarOps> Events for EventsDim1<NTY> {
Box::new(ret) Box::new(ret)
} }
fn move_into_fresh(&mut self, ts_end: u64) -> Box<dyn Events> {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
let tss = self.tss.drain(..n1).collect();
let pulses = self.pulses.drain(..n1).collect();
let values = self.values.drain(..n1).collect();
let ret = Self { tss, pulses, values };
Box::new(ret)
}
fn move_into_existing(&mut self, tgt: &mut Box<dyn Events>, ts_end: u64) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrealted traits. Simplify.
if let Some(tgt) = tgt.as_mut().as_any_mut().downcast_mut::<Self>() {
// TODO improve the search
let n1 = self.tss.iter().take_while(|&&x| x <= ts_end).count();
// TODO make it harder to forget new members when the struct may get modified in the future
tgt.tss.extend(self.tss.drain(..n1));
tgt.pulses.extend(self.pulses.drain(..n1));
tgt.values.extend(self.values.drain(..n1));
Ok(())
} else {
eprintln!("downcast to EventsDim0 FAILED");
Err(())
}
}
fn new_empty(&self) -> Box<dyn Events> { fn new_empty(&self) -> Box<dyn Events> {
Box::new(Self::empty()) Box::new(Self::empty())
} }

View File

@@ -182,16 +182,6 @@ impl crate::merger::Mergeable for Box<dyn Events> {
self.as_ref().ts_max() self.as_ref().ts_max()
} }
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
self.as_mut().move_into_fresh(ts_end)
}
fn move_into_existing(&mut self, tgt: &mut Self, ts_end: u64) -> Result<(), merger::MergeError> {
self.as_mut()
.move_into_existing(tgt, ts_end)
.map_err(|()| merger::MergeError::NotCompatible)
}
fn new_empty(&self) -> Self { fn new_empty(&self) -> Self {
self.as_ref().new_empty() self.as_ref().new_empty()
} }

View File

@@ -1,13 +1,17 @@
use crate::Error; use crate::Error;
use futures_util::{Stream, StreamExt}; use futures_util::Stream;
use futures_util::StreamExt;
use items::sitem_data; use items::sitem_data;
use items::{RangeCompletableItem, Sitemty, StreamItem}; use items::RangeCompletableItem;
use items::Sitemty;
use items::StreamItem;
use netpod::log::*; use netpod::log::*;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::fmt; use std::fmt;
use std::ops::{ControlFlow, RangeBounds}; use std::ops::ControlFlow;
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::Context;
use std::task::Poll;
#[allow(unused)] #[allow(unused)]
macro_rules! trace2 { macro_rules! trace2 {
@@ -43,12 +47,6 @@ pub trait Mergeable<Rhs = Self>: fmt::Debug + Unpin {
fn len(&self) -> usize; fn len(&self) -> usize;
fn ts_min(&self) -> Option<u64>; fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>; fn ts_max(&self) -> Option<u64>;
// TODO remove, superseded.
fn move_into_fresh(&mut self, ts_end: u64) -> Rhs;
fn move_into_existing(&mut self, tgt: &mut Rhs, ts_end: u64) -> Result<(), MergeError>;
// TODO: split the logic into: make fresh container, and a single drain_into method. Or is there any advantage in having both?
fn new_empty(&self) -> Self; fn new_empty(&self) -> Self;
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError>; fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError>;
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>; fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>;

View File

@@ -1,18 +1,33 @@
use crate::binned_collected;
use crate::binsdim0::BinsDim0CollectedResult; use crate::binsdim0::BinsDim0CollectedResult;
use crate::channelevents::{ConnStatus, ConnStatusEvent}; use crate::channelevents::ConnStatus;
use crate::channelevents::ConnStatusEvent;
use crate::eventsdim0::EventsDim0; use crate::eventsdim0::EventsDim0;
use crate::merger::{Mergeable, Merger}; use crate::merger::Mergeable;
use crate::merger::Merger;
use crate::merger_cev::ChannelEventsMerger; use crate::merger_cev::ChannelEventsMerger;
use crate::runfut;
use crate::testgen::make_some_boxed_d0_f32; use crate::testgen::make_some_boxed_d0_f32;
use crate::ChannelEvents;
use crate::Error; use crate::Error;
use crate::{binned_collected, runfut, ChannelEvents, Events, IsoDateTime}; use crate::Events;
use chrono::{TimeZone, Utc}; use crate::IsoDateTime;
use futures_util::{stream, StreamExt}; use chrono::TimeZone;
use items::{sitem_data, RangeCompletableItem, Sitemty, StreamItem}; use chrono::Utc;
use futures_util::stream;
use futures_util::StreamExt;
use items::sitem_data;
use items::RangeCompletableItem;
use items::Sitemty;
use items::StreamItem;
use items_0::Empty; use items_0::Empty;
use netpod::log::*; use netpod::log::*;
use netpod::timeunits::*; use netpod::timeunits::*;
use netpod::{AggKind, BinnedRange, NanoRange, ScalarType, Shape}; use netpod::AggKind;
use netpod::BinnedRange;
use netpod::NanoRange;
use netpod::ScalarType;
use netpod::Shape;
use std::time::Duration; use std::time::Duration;
#[test] #[test]
@@ -22,10 +37,25 @@ fn items_move_events() {
let mut v1 = v0.clone(); let mut v1 = v0.clone();
eprintln!("{v1:?}"); eprintln!("{v1:?}");
eprintln!("{}", v1.len()); eprintln!("{}", v1.len());
let mut v2 = v1.move_into_fresh(4); let mut v2 = v1.new_empty();
match v1.find_lowest_index_gt(4) {
Some(ilgt) => {
v1.drain_into(&mut v2, (0, ilgt)).unwrap();
}
None => {
v1.drain_into(&mut v2, (0, v1.len())).unwrap();
}
}
eprintln!("{}", v1.len()); eprintln!("{}", v1.len());
eprintln!("{}", v2.len()); eprintln!("{}", v2.len());
v1.move_into_existing(&mut v2, u64::MAX).unwrap(); match v1.find_lowest_index_gt(u64::MAX) {
Some(ilgt) => {
v1.drain_into(&mut v2, (0, ilgt)).unwrap();
}
None => {
v1.drain_into(&mut v2, (0, v1.len())).unwrap();
}
}
eprintln!("{}", v1.len()); eprintln!("{}", v1.len());
eprintln!("{}", v2.len()); eprintln!("{}", v2.len());
eprintln!("{v1:?}"); eprintln!("{v1:?}");
@@ -62,9 +92,9 @@ fn items_merge_01() {
let evs1 = make_some_boxed_d0_f32(10, SEC * 2, SEC * 2, 0, 828764893); let evs1 = make_some_boxed_d0_f32(10, SEC * 2, SEC * 2, 0, 828764893);
let v0 = ChannelEvents::Events(evs0); let v0 = ChannelEvents::Events(evs0);
let v1 = ChannelEvents::Events(evs1); let v1 = ChannelEvents::Events(evs1);
let v2 = ChannelEvents::Status(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)); let v2 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)));
let v3 = ChannelEvents::Status(ConnStatusEvent::new(MS * 2300, ConnStatus::Disconnect)); let v3 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 2300, ConnStatus::Disconnect)));
let v4 = ChannelEvents::Status(ConnStatusEvent::new(MS * 2800, ConnStatus::Connect)); let v4 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 2800, ConnStatus::Connect)));
let stream0 = Box::pin(stream::iter(vec![sitem_data(v0)])); let stream0 = Box::pin(stream::iter(vec![sitem_data(v0)]));
let stream1 = Box::pin(stream::iter(vec![sitem_data(v1)])); let stream1 = Box::pin(stream::iter(vec![sitem_data(v1)]));
let stream2 = Box::pin(stream::iter(vec![sitem_data(v2), sitem_data(v3), sitem_data(v4)])); let stream2 = Box::pin(stream::iter(vec![sitem_data(v2), sitem_data(v3), sitem_data(v4)]));
@@ -97,9 +127,9 @@ fn items_merge_02() {
let evs1 = make_some_boxed_d0_f32(100, SEC * 2, SEC * 2, 0, 828764893); let evs1 = make_some_boxed_d0_f32(100, SEC * 2, SEC * 2, 0, 828764893);
let v0 = ChannelEvents::Events(evs0); let v0 = ChannelEvents::Events(evs0);
let v1 = ChannelEvents::Events(evs1); let v1 = ChannelEvents::Events(evs1);
let v2 = ChannelEvents::Status(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)); let v2 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)));
let v3 = ChannelEvents::Status(ConnStatusEvent::new(MS * 2300, ConnStatus::Disconnect)); let v3 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 2300, ConnStatus::Disconnect)));
let v4 = ChannelEvents::Status(ConnStatusEvent::new(MS * 2800, ConnStatus::Connect)); let v4 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 2800, ConnStatus::Connect)));
let stream0 = Box::pin(stream::iter(vec![sitem_data(v0)])); let stream0 = Box::pin(stream::iter(vec![sitem_data(v0)]));
let stream1 = Box::pin(stream::iter(vec![sitem_data(v1)])); let stream1 = Box::pin(stream::iter(vec![sitem_data(v1)]));
let stream2 = Box::pin(stream::iter(vec![sitem_data(v2), sitem_data(v3), sitem_data(v4)])); let stream2 = Box::pin(stream::iter(vec![sitem_data(v2), sitem_data(v3), sitem_data(v4)]));
@@ -241,7 +271,7 @@ fn merge03() {
status: ConnStatus::Disconnect, status: ConnStatus::Disconnect,
}; };
let item: Sitemty<ChannelEvents> = Ok(StreamItem::DataItem(RangeCompletableItem::Data( let item: Sitemty<ChannelEvents> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(
ChannelEvents::Status(ev), ChannelEvents::Status(Some(ev)),
))); )));
vec![item] vec![item]
}; };
@@ -253,7 +283,7 @@ fn merge03() {
status: ConnStatus::Disconnect, status: ConnStatus::Disconnect,
}; };
let item: Sitemty<ChannelEvents> = Ok(StreamItem::DataItem(RangeCompletableItem::Data( let item: Sitemty<ChannelEvents> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(
ChannelEvents::Status(ev), ChannelEvents::Status(Some(ev)),
))); )));
vec![item] vec![item]
}; };

View File

@@ -17,6 +17,7 @@ arrayref = "0.3.6"
crc32fast = "1.3.2" crc32fast = "1.3.2"
byteorder = "1.4.3" byteorder = "1.4.3"
chrono = { version = "0.4.19", features = ["serde"] } chrono = { version = "0.4.19", features = ["serde"] }
wasmer = { version = "3.1.1", default-features = false, features = ["sys", "cranelift"] }
err = { path = "../err" } err = { path = "../err" }
netpod = { path = "../netpod" } netpod = { path = "../netpod" }
items = { path = "../items" } items = { path = "../items" }

View File

@@ -17,6 +17,17 @@ pub async fn plain_events_json(
chconf: &ChConf, chconf: &ChConf,
cluster: &Cluster, cluster: &Cluster,
) -> Result<JsonValue, Error> { ) -> Result<JsonValue, Error> {
if query.channel().name() == "wasm-test-01" {
use wasmer::Value;
let wasm = query.channel().name().as_bytes();
let mut store = wasmer::Store::default();
let module = wasmer::Module::new(&store, wasm).unwrap();
let import_object = wasmer::imports! {};
let instance = wasmer::Instance::new(&mut store, &module, &import_object).unwrap();
let add_one = instance.exports.get_function("event_transform").unwrap();
let result = add_one.call(&mut store, &[Value::I32(42)]).unwrap();
assert_eq!(result[0], Value::I32(43));
}
// TODO remove magic constant // TODO remove magic constant
let deadline = Instant::now() + query.timeout() + Duration::from_millis(1000); let deadline = Instant::now() + query.timeout() + Duration::from_millis(1000);
let events_max = query.events_max(); let events_max = query.events_max();

View File

@@ -16,8 +16,8 @@ fn time_bin_00() {
let edges = [0, 1, 2, 3, 4, 5, 6, 7, 8].into_iter().map(|x| SEC * x).collect(); let edges = [0, 1, 2, 3, 4, 5, 6, 7, 8].into_iter().map(|x| SEC * x).collect();
let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782); let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782);
let v0 = ChannelEvents::Events(evs0); let v0 = ChannelEvents::Events(evs0);
let v2 = ChannelEvents::Status(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)); let v2 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)));
let v4 = ChannelEvents::Status(ConnStatusEvent::new(MS * 6000, ConnStatus::Disconnect)); let v4 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 6000, ConnStatus::Disconnect)));
let stream0 = Box::pin(stream::iter(vec![ let stream0 = Box::pin(stream::iter(vec![
// //
sitem_data(v2), sitem_data(v2),