WIP typechecks

This commit is contained in:
Dominik Werder
2024-10-22 16:14:32 +02:00
parent 773da33777
commit f754c5c962
36 changed files with 631 additions and 273 deletions

View File

@@ -2,11 +2,10 @@ use super::aggregator::AggregatorNumeric;
use super::aggregator::AggregatorTimeWeight;
use super::container_events::EventValueType;
use super::___;
use crate::vecpreview::PreviewRange;
use crate::vecpreview::VecPreview;
use core::fmt;
use err::thiserror;
use err::ThisError;
use items_0::vecpreview::VecPreview;
use netpod::TsNano;
use serde::Deserialize;
use serde::Serialize;
@@ -114,6 +113,28 @@ impl<EVT> ContainerBins<EVT>
where
EVT: EventValueType,
{
pub fn from_constituents(
ts1s: VecDeque<TsNano>,
ts2s: VecDeque<TsNano>,
cnts: VecDeque<u64>,
mins: VecDeque<EVT>,
maxs: VecDeque<EVT>,
avgs: VecDeque<EVT::AggTimeWeightOutputAvg>,
lsts: VecDeque<EVT>,
fnls: VecDeque<bool>,
) -> Self {
Self {
ts1s,
ts2s,
cnts,
mins,
maxs,
avgs,
lsts,
fnls,
}
}
pub fn type_name() -> &'static str {
any::type_name::<Self>()
}
@@ -153,6 +174,14 @@ where
self.ts2s.back().map(|&x| x)
}
pub fn ts1s_iter(&self) -> std::collections::vec_deque::Iter<TsNano> {
self.ts1s.iter()
}
pub fn ts2s_iter(&self) -> std::collections::vec_deque::Iter<TsNano> {
self.ts2s.iter()
}
pub fn cnts_iter(&self) -> std::collections::vec_deque::Iter<u64> {
self.cnts.iter()
}
@@ -165,6 +194,50 @@ where
self.maxs.iter()
}
pub fn avgs_iter(&self) -> std::collections::vec_deque::Iter<EVT::AggTimeWeightOutputAvg> {
self.avgs.iter()
}
pub fn fnls_iter(&self) -> std::collections::vec_deque::Iter<bool> {
self.fnls.iter()
}
pub fn zip_iter(
&self,
) -> std::iter::Zip<
std::iter::Zip<
std::iter::Zip<
std::iter::Zip<
std::iter::Zip<
std::iter::Zip<
std::collections::vec_deque::Iter<TsNano>,
std::collections::vec_deque::Iter<TsNano>,
>,
std::collections::vec_deque::Iter<u64>,
>,
std::collections::vec_deque::Iter<EVT>,
>,
std::collections::vec_deque::Iter<EVT>,
>,
std::collections::vec_deque::Iter<EVT::AggTimeWeightOutputAvg>,
>,
std::collections::vec_deque::Iter<bool>,
> {
self.ts1s_iter()
.zip(self.ts2s_iter())
.zip(self.cnts_iter())
.zip(self.mins_iter())
.zip(self.maxs_iter())
.zip(self.avgs_iter())
.zip(self.fnls_iter())
}
pub fn edges_iter(
&self,
) -> std::iter::Zip<std::collections::vec_deque::Iter<TsNano>, std::collections::vec_deque::Iter<TsNano>> {
self.ts1s.iter().zip(self.ts2s.iter())
}
pub fn len_before(&self, end: TsNano) -> usize {
let pp = self.ts2s.partition_point(|&x| x <= end);
assert!(pp <= self.len(), "len_before pp {} len {}", pp, self.len());

View File

@@ -2,11 +2,12 @@ use super::aggregator::AggTimeWeightOutputAvg;
use super::aggregator::AggregatorNumeric;
use super::aggregator::AggregatorTimeWeight;
use super::___;
use crate::vecpreview::PreviewRange;
use crate::vecpreview::VecPreview;
use core::fmt;
use err::thiserror;
use err::ThisError;
use items_0::timebin::BinningggContainerEventsDyn;
use items_0::vecpreview::PreviewRange;
use items_0::vecpreview::VecPreview;
use netpod::TsNano;
use serde::Deserialize;
use serde::Serialize;
@@ -20,7 +21,7 @@ macro_rules! trace_init { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[cstm(name = "ValueContainerError")]
pub enum ValueContainerError {}
pub trait Container<EVT>: fmt::Debug + Clone + PreviewRange + Serialize + for<'a> Deserialize<'a> {
pub trait Container<EVT>: fmt::Debug + Send + Clone + PreviewRange + Serialize + for<'a> Deserialize<'a> {
fn new() -> Self;
// fn verify(&self) -> Result<(), ValueContainerError>;
fn push_back(&mut self, val: EVT);
@@ -120,6 +121,10 @@ impl<EVT> ContainerEvents<EVT>
where
EVT: EventValueType,
{
pub fn from_constituents(tss: VecDeque<TsNano>, vals: <EVT as EventValueType>::Container) -> Self {
Self { tss, vals }
}
pub fn type_name() -> &'static str {
any::type_name::<Self>()
}
@@ -233,3 +238,12 @@ where
}
}
}
impl<EVT> BinningggContainerEventsDyn for ContainerEvents<EVT>
where
EVT: EventValueType,
{
fn binned_events_timeweight_traitobj(&self) -> Box<dyn items_0::timebin::BinnedEventsTimeweightTrait> {
todo!()
}
}

View File

@@ -1,4 +1,5 @@
pub mod timeweight_bins;
pub mod timeweight_bins_dyn;
pub mod timeweight_events;
pub mod timeweight_events_dyn;

View File

@@ -0,0 +1,27 @@
use futures_util::Stream;
use items_0::streamitem::Sitemty;
use items_0::timebin::BinningggContainerBinsDyn;
use netpod::BinnedRange;
use netpod::TsNano;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
pub struct BinnedBinsTimeweightStream {}
impl BinnedBinsTimeweightStream {
pub fn new(
range: BinnedRange<TsNano>,
inp: Pin<Box<dyn Stream<Item = Sitemty<Box<dyn BinningggContainerBinsDyn>>> + Send>>,
) -> Self {
todo!()
}
}
impl Stream for BinnedBinsTimeweightStream {
type Item = Sitemty<Box<dyn BinningggContainerBinsDyn>>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
todo!()
}
}

View File

@@ -19,6 +19,7 @@ use netpod::DtNano;
use netpod::TsNano;
use std::collections::VecDeque;
use std::marker::PhantomData;
use std::mem;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -382,6 +383,7 @@ where
range: BinnedRange<TsNano>,
inner_a: InnerA<EVT>,
out: ContainerBins<EVT>,
produce_cnt_zero: bool,
}
impl<EVT> fmt::Debug for BinnedEventsTimeweight<EVT>
@@ -422,9 +424,16 @@ where
},
lst: None,
out: ContainerBins::new(),
produce_cnt_zero: true,
}
}
pub fn disable_cnt_zero(self) -> Self {
let mut ret = self;
ret.produce_cnt_zero = false;
ret
}
fn ingest_event_without_lst(&mut self, ev: EventSingle<EVT>) -> Result<(), Error> {
let b = &self.inner_a.inner_b;
if ev.ts >= b.active_end {
@@ -485,10 +494,26 @@ where
let div = b.active_len.ns();
if let Some(lst) = self.lst.as_ref() {
let lst = LstRef(lst);
let mut i = 0;
loop {
i += 1;
assert!(i < 100000, "too many iterations");
if self.produce_cnt_zero {
let mut i = 0;
loop {
i += 1;
assert!(i < 100000, "too many iterations");
let b = &self.inner_a.inner_b;
if ts > b.filled_until {
if ts >= b.active_end {
if b.filled_until < b.active_end {
self.inner_a.inner_b.fill_until(b.active_end, lst.clone());
}
self.inner_a.push_out_and_reset(lst.clone(), true, &mut self.out);
} else {
self.inner_a.inner_b.fill_until(ts, lst.clone());
}
} else {
break;
}
}
} else {
let b = &self.inner_a.inner_b;
if ts > b.filled_until {
if ts >= b.active_end {
@@ -497,13 +522,29 @@ where
}
self.inner_a.push_out_and_reset(lst.clone(), true, &mut self.out);
} else {
// TODO should not hit this case. Prove it, assert it.
self.inner_a.inner_b.fill_until(ts, lst.clone());
}
} else {
break;
// TODO should never hit this case. Count.
}
// TODO jump to next bin
// TODO merge with the other reset
// Below uses the same code
let ts1 = TsNano::from_ns(ts.ns() / div * div);
let b = &mut self.inner_a.inner_b;
b.active_beg = ts1;
b.active_end = ts1.add_dt_nano(b.active_len);
b.filled_until = ts1;
b.filled_width = DtNano::from_ns(0);
b.cnt = 0;
b.agg.reset_for_new_bin();
// assert!(self.inner_a.minmax.is_none());
trace_cycle!("cycled direct to {:?} {:?}", b.active_beg, b.active_end);
}
} else {
assert!(self.inner_a.minmax.is_none());
// TODO merge with the other reset
let ts1 = TsNano::from_ns(ts.ns() / div * div);
let b = &mut self.inner_a.inner_b;
@@ -513,7 +554,6 @@ where
b.filled_width = DtNano::from_ns(0);
b.cnt = 0;
b.agg.reset_for_new_bin();
assert!(self.inner_a.minmax.is_none());
trace_cycle!("cycled direct to {:?} {:?}", b.active_beg, b.active_end);
}
}
@@ -594,6 +634,6 @@ where
}
pub fn output(&mut self) -> ContainerBins<EVT> {
::core::mem::replace(&mut self.out, ContainerBins::new())
mem::replace(&mut self.out, ContainerBins::new())
}
}

View File

@@ -1,9 +1,11 @@
use super::timeweight_events::BinnedEventsTimeweight;
use crate::binning::container_bins::ContainerBins;
use crate::binning::container_events::EventValueType;
use crate::channelevents::ChannelEvents;
use err::thiserror;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::Sitemty;
use items_0::timebin::BinnedEventsTimeweightTrait;
use items_0::timebin::BinningggBinnerDyn;
@@ -12,6 +14,8 @@ use items_0::timebin::BinningggContainerEventsDyn;
use items_0::timebin::BinningggError;
use netpod::BinnedRange;
use netpod::TsNano;
use std::arch::x86_64;
use std::ops::ControlFlow;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -68,18 +72,6 @@ where
}
}
pub struct BinnedEventsTimeweightStream {
inp: Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>,
}
impl Stream for BinnedEventsTimeweightStream {
type Item = ();
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
todo!()
}
}
#[derive(Debug)]
pub struct BinnedEventsTimeweightLazy {
range: BinnedRange<TsNano>,
@@ -114,3 +106,109 @@ impl BinnedEventsTimeweightTrait for BinnedEventsTimeweightLazy {
todo!()
}
}
enum StreamState {
Reading,
Done,
}
pub struct BinnedEventsTimeweightStream {
state: StreamState,
inp: Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>,
binned_events: BinnedEventsTimeweightLazy,
range_complete: bool,
}
impl BinnedEventsTimeweightStream {
pub fn new(range: BinnedRange<TsNano>, inp: Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>) -> Self {
Self {
state: StreamState::Reading,
inp,
binned_events: BinnedEventsTimeweightLazy::new(range),
range_complete: false,
}
}
fn handle_sitemty(
mut self: Pin<&mut Self>,
item: Sitemty<ChannelEvents>,
cx: &mut Context,
) -> ControlFlow<Poll<Option<<Self as Stream>::Item>>> {
use items_0::streamitem::RangeCompletableItem::*;
use items_0::streamitem::StreamItem::*;
use ControlFlow::*;
use Poll::*;
match item {
Ok(x) => match x {
DataItem(x) => match x {
Data(x) => match x {
ChannelEvents::Events(evs) => match self.binned_events.ingest(evs.to_container_events()) {
Ok(()) => {
match self.binned_events.output() {
Ok(x) => {
if x.len() == 0 {
Continue(())
} else {
Break(Ready(Some(Ok(DataItem(Data(x))))))
}
}
Err(e) => Break(Ready(Some(Err(::err::Error::from_string(e))))),
}
// Continue(())
}
Err(e) => Break(Ready(Some(Err(::err::Error::from_string(e))))),
},
ChannelEvents::Status(_) => {
// TODO use the status
Continue(())
}
},
RangeComplete => {
self.range_complete = true;
Continue(())
}
},
Log(x) => Break(Ready(Some(Ok(Log(x))))),
Stats(x) => Break(Ready(Some(Ok(Stats(x))))),
},
Err(e) => {
self.state = StreamState::Done;
Break(Ready(Some(Err(e))))
}
}
}
fn handle_eos(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<<Self as Stream>::Item>> {
use items_0::streamitem::RangeCompletableItem::*;
use items_0::streamitem::StreamItem::*;
use Poll::*;
if self.range_complete {
self.binned_events.input_done_range_final();
} else {
self.binned_events.input_done_range_open();
}
match self.binned_events.output() {
Ok(x) => Ready(Some(Ok(DataItem(Data(x))))),
Err(e) => Ready(Some(Err(::err::Error::from_string(e)))),
}
}
}
impl Stream for BinnedEventsTimeweightStream {
type Item = Sitemty<Box<dyn BinningggContainerBinsDyn>>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use ControlFlow::*;
use Poll::*;
loop {
break match self.as_mut().inp.poll_next_unpin(cx) {
Ready(Some(x)) => match self.as_mut().handle_sitemty(x, cx) {
Continue(()) => continue,
Break(x) => x,
},
Ready(None) => self.handle_eos(cx),
Pending => Pending,
};
}
}
}

View File

@@ -2,8 +2,8 @@ use super::aggregator::AggregatorTimeWeight;
use super::binnedvaluetype::BinnedNumericValue;
use super::container_events::Container;
use super::container_events::EventValueType;
use crate::vecpreview::PreviewRange;
use core::fmt;
use items_0::vecpreview::PreviewRange;
use netpod::DtNano;
use netpod::EnumVariant;
use serde::Deserialize;
@@ -18,7 +18,7 @@ pub struct EnumVariantContainer {
impl PreviewRange for EnumVariantContainer {
fn preview<'a>(&'a self) -> Box<dyn fmt::Debug + 'a> {
let ret = crate::vecpreview::PreviewCell {
let ret = items_0::vecpreview::PreviewCell {
a: self.ixs.front(),
b: self.ixs.back(),
};

View File

@@ -2,7 +2,6 @@ use crate::timebin::TimeBinnerCommonV0Func;
use crate::timebin::TimeBinnerCommonV0Trait;
use crate::ts_offs_from_abs;
use crate::ts_offs_from_abs_with_anchor;
use crate::vecpreview::VecPreview;
use crate::IsoDateTime;
use crate::RangeOverlapInfo;
use crate::TimeBinnableType;
@@ -25,6 +24,7 @@ use items_0::timebin::TimeBinned;
use items_0::timebin::TimeBinner;
use items_0::timebin::TimeBinnerTy;
use items_0::timebin::TimeBins;
use items_0::vecpreview::VecPreview;
use items_0::AppendAllFrom;
use items_0::AppendEmptyBin;
use items_0::AsAnyMut;
@@ -134,6 +134,10 @@ where
impl<NTY: ScalarOps> BinsDim0<NTY> {
pub fn push(&mut self, ts1: u64, ts2: u64, count: u64, min: NTY, max: NTY, avg: f32, lst: NTY) {
if avg < min.as_prim_f32_b() || avg > max.as_prim_f32_b() {
// TODO rounding issues?
debug!("bad avg");
}
self.ts1s.push_back(ts1);
self.ts2s.push_back(ts2);
self.cnts.push_back(count);
@@ -534,12 +538,19 @@ where
if self.cnt == 0 && !push_empty {
self.reset_agg();
} else {
let min = self.min.clone();
let max = self.max.clone();
let avg = self.avg as f32;
if avg < min.as_prim_f32_b() || avg > max.as_prim_f32_b() {
// TODO rounding issues?
debug!("bad avg");
}
self.out.ts1s.push_back(self.ts1now.ns());
self.out.ts2s.push_back(self.ts2now.ns());
self.out.cnts.push_back(self.cnt);
self.out.mins.push_back(self.min.clone());
self.out.maxs.push_back(self.max.clone());
self.out.avgs.push_back(self.avg as f32);
self.out.mins.push_back(min);
self.out.maxs.push_back(max);
self.out.avgs.push_back(avg);
self.out.lsts.push_back(self.lst.clone());
self.reset_agg();
}

View File

@@ -1023,6 +1023,10 @@ impl Events for ChannelEvents {
Status(x) => panic!("ChannelEvents::to_dim0_f32_for_binning"),
}
}
fn to_container_events(&self) -> Box<dyn ::items_0::timebin::BinningggContainerEventsDyn> {
panic!("should not get used")
}
}
impl Collectable for ChannelEvents {

View File

@@ -270,6 +270,7 @@ pub enum DecompError {
UnusedBytes,
BitshuffleError,
ShapeMakesNoSense,
UnexpectedCompressedScalarValue,
}
fn decompress(databuf: &[u8], type_size: u32) -> Result<Vec<u8>, DecompError> {
@@ -325,10 +326,18 @@ impl EventFull {
/// but we still don't know whether that's an image or a waveform.
/// Therefore, the function accepts the expected shape to at least make an assumption
/// about whether this is an image or a waveform.
pub fn shape_derived(&self, i: usize, shape_exp: &Shape) -> Result<Shape, DecompError> {
pub fn shape_derived(
&self,
i: usize,
scalar_type_exp: &ScalarType,
shape_exp: &Shape,
) -> Result<Shape, DecompError> {
match shape_exp {
Shape::Scalar => match &self.comps[i] {
Some(_) => Err(DecompError::ShapeMakesNoSense),
Some(_) => match scalar_type_exp {
ScalarType::STRING => Ok(Shape::Scalar),
_ => Err(DecompError::UnexpectedCompressedScalarValue),
},
None => Ok(Shape::Scalar),
},
Shape::Wave(_) => match &self.shapes[i] {

View File

@@ -760,6 +760,31 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
} else {
lst.as_prim_f32_b()
};
let max = if min > max {
// TODO count
debug!("min > max");
min.clone()
} else {
max
};
let avg = {
let g = min.as_prim_f32_b();
if avg < g {
debug!("avg < min");
g
} else {
avg
}
};
let avg = {
let g = max.as_prim_f32_b();
if avg > g {
debug!("avg > max");
g
} else {
avg
}
};
let ret = if self.range.is_time() {
BinsDim0 {
ts1s: [range_beg].into(),
@@ -1078,6 +1103,14 @@ impl<STY: ScalarOps> Events for EventsDim0<STY> {
}
Box::new(ret)
}
fn to_container_events(&self) -> Box<dyn ::items_0::timebin::BinningggContainerEventsDyn> {
// let tss = self.tss.iter().map(|&x| TsNano::from_ns(x)).collect();
// let vals = self.values.clone();
// let ret = crate::binning::container_events::ContainerEvents::from_constituents(tss, vals);
// Box::new(ret)
todo!()
}
}
#[derive(Debug)]

View File

@@ -499,4 +499,8 @@ impl Events for EventsDim0Enum {
fn to_dim0_f32_for_binning(&self) -> Box<dyn Events> {
todo!("{}::to_dim0_f32_for_binning", self.type_name())
}
fn to_container_events(&self) -> Box<dyn ::items_0::timebin::BinningggContainerEventsDyn> {
todo!("{}::to_container_events", self.type_name())
}
}

View File

@@ -993,6 +993,10 @@ impl<STY: ScalarOps> Events for EventsDim1<STY> {
fn to_dim0_f32_for_binning(&self) -> Box<dyn Events> {
todo!("{}::to_dim0_f32_for_binning", self.type_name())
}
fn to_container_events(&self) -> Box<dyn ::items_0::timebin::BinningggContainerEventsDyn> {
todo!("{}::to_container_events", self.type_name())
}
}
#[derive(Debug)]

View File

@@ -381,6 +381,10 @@ impl<STY: ScalarOps> Events for EventsXbinDim0<STY> {
fn to_dim0_f32_for_binning(&self) -> Box<dyn Events> {
todo!("{}::to_dim0_f32_for_binning", self.type_name())
}
fn to_container_events(&self) -> Box<dyn ::items_0::timebin::BinningggContainerEventsDyn> {
todo!("{}::to_container_events", self.type_name())
}
}
#[derive(Debug)]

View File

@@ -1,53 +1 @@
use core::fmt;
use std::collections::VecDeque;
pub struct PreviewCell<'a, T> {
pub a: Option<&'a T>,
pub b: Option<&'a T>,
}
impl<'a, T> fmt::Debug for PreviewCell<'a, T>
where
T: fmt::Debug,
{
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match (self.a.as_ref(), self.b.as_ref()) {
(Some(a), Some(b)) => write!(fmt, "{:?} .. {:?}", a, b),
(Some(a), None) => write!(fmt, "{:?}", a),
_ => write!(fmt, "(empty)"),
}
}
}
pub trait PreviewRange {
fn preview<'a>(&'a self) -> Box<dyn fmt::Debug + 'a>;
}
impl<T> PreviewRange for VecDeque<T>
where
T: fmt::Debug,
{
fn preview<'a>(&'a self) -> Box<dyn fmt::Debug + 'a> {
let ret = PreviewCell {
a: self.front(),
b: if self.len() <= 1 { None } else { self.back() },
};
Box::new(ret)
}
}
pub struct VecPreview<'a> {
c: &'a dyn PreviewRange,
}
impl<'a> VecPreview<'a> {
pub fn new(c: &'a dyn PreviewRange) -> Self {
Self { c }
}
}
impl<'a> fmt::Debug for VecPreview<'a> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{:?}", self.c.preview())
}
}