daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP checks

Browse Source
This commit is contained in:
Dominik Werder
2023-04-05 12:00:18 +02:00
parent 7c9085fe5b
commit 81298b16df
39 changed files with 892 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
streams/src/collect.rs
View File

@@ -47,6 +47,7 @@ where
S: Stream<Item = Sitemty<T>> + Unpin,
T: Collectable + WithLen + fmt::Debug,
{
info!("collect events_max {events_max} deadline {deadline:?}");
let mut collector: Option<Box<dyn Collector>> = None;
let mut stream = stream;
let deadline = deadline.into();
@@ -58,17 +59,16 @@ where
Ok(Some(k)) => k,
Ok(None) => break,
Err(_e) => {
warn!("collect_in_span time out");
warn!("collect timeout");
timed_out = true;
if let Some(coll) = collector.as_mut() {
coll.set_timed_out();
} else {
warn!("Timeout but no collector yet");
warn!("collect timeout but no collector yet");
}
break;
}
};
trace!("collect_in_span see item {item:?}");
match item {
Ok(item) => match item {
StreamItem::DataItem(item) => match item {
@@ -77,11 +77,11 @@ where
if let Some(coll) = collector.as_mut() {
coll.set_range_complete();
} else {
warn!("Received RangeComplete but no collector yet");
warn!("collect received RangeComplete but no collector yet");
}
}
RangeCompletableItem::Data(mut item) => {
debug!("collect_in_span sees len {}", item.len());
info!("collect sees len {}", item.len());
if collector.is_none() {
let c = item.new_collector();
collector = Some(c);
@@ -95,10 +95,10 @@ where
}
},
StreamItem::Log(item) => {
trace!("Log {:?}", item);
trace!("collect log {:?}", item);
}
StreamItem::Stats(item) => {
trace!("Stats {:?}", item);
trace!("collect stats {:?}", item);
match item {
// TODO factor and simplify the stats collection:
StatsItem::EventDataReadStats(_) => {}
@@ -131,7 +131,7 @@ where
let res = collector
.ok_or_else(|| Error::with_msg_no_trace(format!("no result because no collector was created")))?
.result(range, binrange)?;
debug!("Total duration: {:?}", total_duration);
info!("collect stats total duration: {:?}", total_duration);
Ok(res)
}

1
streams/src/eventchunker.rs
View File

@@ -1 +0,0 @@

4
streams/src/frames/eventsfromframes.rs
View File

@@ -57,7 +57,7 @@ where
match self.inp.poll_next_unpin(cx) {
Ready(Some(Ok(item))) => match item {
StreamItem::Log(item) => {
info!("{} {:?} {}", item.node_ix, item.level, item.msg);
//info!("{} {:?} {}", item.node_ix, item.level, item.msg);
Ready(Some(Ok(StreamItem::Log(item))))
}
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
@@ -65,7 +65,7 @@ where
Ok(item) => match item {
Ok(item) => match item {
StreamItem::Log(k) => {
info!("rcvd log: {} {:?} {}", k.node_ix, k.level, k.msg);
//info!("rcvd log: {} {:?} {}", k.node_ix, k.level, k.msg);
Ready(Some(Ok(StreamItem::Log(k))))
}
item => Ready(Some(Ok(item))),

86
streams/src/generators.rs Normal file
View File

@@ -0,0 +1,86 @@
use err::Error;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use items_0::container::ByteEstimate;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use netpod::log::*;
use netpod::range::evrange::SeriesRange;
use netpod::timeunits::MS;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use std::time::Duration;
pub struct GenerateI32 {
ts: u64,
dts: u64,
tsend: u64,
timeout: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
}
impl GenerateI32 {
pub fn new(node_ix: u64, node_count: u64, range: SeriesRange) -> Self {
let range = match range {
SeriesRange::TimeRange(k) => k,
SeriesRange::PulseRange(_) => todo!(),
};
let dts = MS * 1000 * node_count as u64;
let ts = (range.beg / dts + node_ix) * dts;
let tsend = range.end;
Self {
ts,
dts,
tsend,
timeout: None,
}
}
fn make_batch(&mut self) -> Sitemty<ChannelEvents> {
type T = i32;
let mut item = items_2::eventsdim0::EventsDim0::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 200 {
break;
}
let pulse = ts;
item.push(ts, pulse, pulse as T);
ts += self.dts;
}
self.ts = ts;
let w = ChannelEvents::Events(Box::new(item) as _);
let w = sitem_data(w);
w
}
}
impl Stream for GenerateI32 {
type Item = Sitemty<ChannelEvents>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.ts >= self.tsend {
Ready(None)
} else if let Some(fut) = self.timeout.as_mut() {
match fut.poll_unpin(cx) {
Ready(()) => {
self.timeout = None;
Ready(Some(self.make_batch()))
}
Pending => Pending,
}
} else {
self.timeout = Some(Box::pin(tokio::time::sleep(Duration::from_millis(500))));
continue;
};
}
}
}

2
streams/src/lib.rs
View File

@@ -1,8 +1,8 @@
pub mod collect;
pub mod dtflags;
pub mod eventchunker;
pub mod filechunkread;
pub mod frames;
pub mod generators;
pub mod needminbuffer;
pub mod plaineventsjson;
pub mod rangefilter2;

9
streams/src/plaineventsjson.rs
View File

@@ -12,7 +12,6 @@ use netpod::ChConf;
use netpod::Cluster;
use query::api4::events::PlainEventsQuery;
use serde_json::Value as JsonValue;
use std::time::Duration;
use std::time::Instant;
pub async fn plain_events_json(evq: &PlainEventsQuery, chconf: &ChConf, cluster: &Cluster) -> Result<JsonValue, Error> {
@@ -28,7 +27,7 @@ pub async fn plain_events_json(evq: &PlainEventsQuery, chconf: &ChConf, cluster:
assert_eq!(result[0], Value::I32(43));
}
// TODO remove magic constant
let deadline = Instant::now() + evq.timeout() + Duration::from_millis(1000);
let deadline = Instant::now() + evq.timeout();
let events_max = evq.events_max();
let evquery = evq.clone();
info!("plain_events_json evquery {:?}", evquery);
@@ -87,13 +86,15 @@ pub async fn plain_events_json(evq: &PlainEventsQuery, chconf: &ChConf, cluster:
Box::pin(stream)
};
#[cfg(DISABLED)]
let stream = stream.map(|item| {
//info!("item after merge: {item:?}");
info!("item after merge: {item:?}");
item
});
let stream = RangeFilter2::new(stream, evq.range().try_into()?, evquery.one_before_range());
#[cfg(DISABLED)]
let stream = stream.map(|item| {
//info!("item after rangefilter: {item:?}");
info!("item after rangefilter: {item:?}");
item
});
let stream = stream::iter([empty]).chain(stream);

5
streams/src/tcprawclient.rs
View File

@@ -9,7 +9,6 @@ use crate::frames::eventsfromframes::EventsFromFrames;
use crate::frames::inmem::InMemoryFrameAsyncReadStream;
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::sitem_data;
use items_0::streamitem::Sitemty;
@@ -80,10 +79,6 @@ where
let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
let stream = stream.map(|x| {
info!("tcp stream recv sees item {x:?}");
x
});
streams.push(Box::pin(stream) as _);
}
Ok(streams)

80
streams/src/test/timebin.rs
View File

@@ -1,18 +1,27 @@
use crate::collect::collect;
use crate::generators::GenerateI32;
use crate::test::runfut;
use chrono::DateTime;
use chrono::Utc;
use err::Error;
use futures_util::stream;
use futures_util::StreamExt;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::StreamItem;
use items_0::timebin::TimeBinnable;
use items_0::Empty;
use items_2::binsdim0::BinsDim0;
use items_2::channelevents::ChannelEvents;
use items_2::channelevents::ConnStatus;
use items_2::channelevents::ConnStatusEvent;
use items_2::testgen::make_some_boxed_d0_f32;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange;
use netpod::timeunits::MS;
use netpod::timeunits::SEC;
use netpod::BinnedRangeEnum;
use std::collections::VecDeque;
use std::time::Duration;
use std::time::Instant;
@@ -20,7 +29,10 @@ use std::time::Instant;
#[test]
fn time_bin_00() {
let fut = async {
let edges = [0, 1, 2, 3, 4, 5, 6, 7, 8].into_iter().map(|x| SEC * x).collect();
let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?;
let range = SeriesRange::TimeRange(range);
let min_bin_count = 8;
let binned_range = BinnedRangeEnum::covering_range(range, min_bin_count)?;
let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782);
let v0 = ChannelEvents::Events(evs0);
let v2 = ChannelEvents::Status(Some(ConnStatusEvent::new(MS * 100, ConnStatus::Connect)));
@@ -46,7 +58,7 @@ fn time_bin_00() {
d
};
let deadline = Instant::now() + Duration::from_millis(2000000);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, edges, true, deadline);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true, deadline);
while let Some(item) = binned_stream.next().await {
//eprintln!("{item:?}");
match item {
@@ -78,7 +90,10 @@ fn time_bin_00() {
#[test]
fn time_bin_01() {
let fut = async {
let edges = [0, 1, 2, 3, 4, 5, 6, 7, 8].into_iter().map(|x| SEC * x).collect();
let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?;
let range = SeriesRange::TimeRange(range);
let min_bin_count = 8;
let binned_range = BinnedRangeEnum::covering_range(range, min_bin_count)?;
let evs0 = make_some_boxed_d0_f32(10, SEC * 1, MS * 500, 0, 1846713782);
let evs1 = make_some_boxed_d0_f32(10, SEC * 6, MS * 500, 0, 1846713781);
let v0 = ChannelEvents::Events(evs0);
@@ -102,7 +117,7 @@ fn time_bin_01() {
});
let stream0 = Box::pin(stream0);
let deadline = Instant::now() + Duration::from_millis(200);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, edges, true, deadline);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true, deadline);
while let Some(item) = binned_stream.next().await {
if true {
eprintln!("{item:?}");
@@ -133,4 +148,61 @@ fn time_bin_01() {
runfut(fut).unwrap()
}
fn nano_range_from_str(beg_date: &str, end_date: &str) -> Result<NanoRange, Error> {
let beg_date = beg_date.parse()?;
let end_date = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date);
Ok(range)
}
#[test]
fn time_bin_02() -> Result<(), Error> {
let fut = async {
let do_time_weight = true;
let deadline = Instant::now() + Duration::from_millis(4000);
let range = nano_range_from_str("1970-01-01T00:20:04Z", "1970-01-01T00:21:10Z")?;
let range = SeriesRange::TimeRange(range);
let min_bin_count = 10;
let binned_range = BinnedRangeEnum::covering_range(range.clone(), min_bin_count)?;
eprintln!("binned_range: {:?}", binned_range);
for i in 0.. {
if let Some(r) = binned_range.range_at(i) {
eprintln!("Series Range to cover: {r:?}");
} else {
break;
}
}
// TODO the test stream must be able to generate also one-before (on demand) and RangeComplete (by default).
let stream = GenerateI32::new(0, 1, range);
// TODO apply first some box dyn EventTransform which later is provided by TransformQuery.
// Then the Merge will happen always by default for backends where this is needed.
// TODO then apply the transform chain for the after-merged-stream.
let stream = stream.map(|x| {
//
on_sitemty_data!(x, |x| Ok(StreamItem::DataItem(RangeCompletableItem::Data(
Box::new(x) as Box<dyn TimeBinnable>
))))
});
let stream = Box::pin(stream);
let mut binned_stream =
crate::timebin::TimeBinnedStream::new(stream, binned_range.clone(), do_time_weight, deadline);
// From there on it should no longer be neccessary to distinguish whether its still events or time bins.
// Then, optionally collect for output type like json, or stream as batches.
// TODO the timebinner should already provide batches to make this efficient.
while let Some(e) = binned_stream.next().await {
eprintln!("see item {e:?}");
let x = on_sitemty_data!(e, |e| {
//
Ok(StreamItem::DataItem(RangeCompletableItem::Data(e)))
});
}
/*let res = collect(binned_stream, deadline, 200, None, Some(binned_range)).await?;
let d = res.to_json_result()?.to_json_bytes()?;
let s = String::from_utf8_lossy(&d);
eprintln!("{s}");*/
Ok(())
};
runfut(fut)
}
// TODO add test case to observe RangeComplete after binning.

41
streams/src/timebin.rs
View File

@@ -7,9 +7,14 @@ use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_2::timebin::TimeBinnable;
use items_2::timebin::TimeBinner;
use items_0::timebin::TimeBinnableTy;
use items_0::timebin::TimeBinner;
use items_0::timebin::TimeBinnerTy;
use netpod::log::*;
use netpod::BinnedRange;
use netpod::BinnedRangeEnum;
use netpod::Dim0Index;
use std::any;
use std::fmt;
use std::pin::Pin;
use std::task::Context;
@@ -18,19 +23,19 @@ use std::time::Instant;
#[allow(unused)]
macro_rules! trace2 {
(D$($arg:tt)*) => ();
(__$($arg:tt)*) => ();
($($arg:tt)*) => (trace!($($arg)*));
}
#[allow(unused)]
macro_rules! trace3 {
(D$($arg:tt)*) => ();
(__$($arg:tt)*) => ();
($($arg:tt)*) => (trace!($($arg)*));
}
#[allow(unused)]
macro_rules! trace4 {
(D$($arg:tt)*) => ();
(__$($arg:tt)*) => ();
($($arg:tt)*) => (trace!($($arg)*));
}
@@ -38,15 +43,15 @@ type MergeInp<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
pub struct TimeBinnedStream<T>
where
T: TimeBinnable,
T: TimeBinnableTy,
{
inp: MergeInp<T>,
edges: Vec<u64>,
range: BinnedRangeEnum,
do_time_weight: bool,
deadline: Instant,
deadline_fut: Pin<Box<dyn Future<Output = ()> + Send>>,
range_complete: bool,
binner: Option<<T as TimeBinnable>::TimeBinner>,
binner: Option<<T as TimeBinnableTy>::TimeBinner>,
done_data: bool,
done: bool,
complete: bool,
@@ -54,11 +59,11 @@ where
impl<T> fmt::Debug for TimeBinnedStream<T>
where
T: TimeBinnable,
T: TimeBinnableTy,
{
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt.debug_struct("TimeBinnedStream")
.field("edges", &self.edges)
fmt.debug_struct(any::type_name::<Self>())
.field("range", &self.range)
.field("deadline", &self.deadline)
.field("range_complete", &self.range_complete)
.field("binner", &self.binner)
@@ -68,14 +73,14 @@ where
impl<T> TimeBinnedStream<T>
where
T: TimeBinnable,
T: TimeBinnableTy,
{
pub fn new(inp: MergeInp<T>, edges: Vec<u64>, do_time_weight: bool, deadline: Instant) -> Self {
pub fn new(inp: MergeInp<T>, range: BinnedRangeEnum, do_time_weight: bool, deadline: Instant) -> Self {
let deadline_fut = tokio::time::sleep_until(deadline.into());
let deadline_fut = Box::pin(deadline_fut);
Self {
inp,
edges,
range,
do_time_weight,
deadline,
deadline_fut,
@@ -91,7 +96,7 @@ where
trace!("process_item {item:?}");
if self.binner.is_none() {
trace!("process_item call time_binner_new");
let binner = item.time_binner_new(todo!(), self.do_time_weight);
let binner = item.time_binner_new(self.range.clone(), self.do_time_weight);
self.binner = Some(binner);
}
let binner = self.binner.as_mut().unwrap();
@@ -102,13 +107,13 @@ where
impl<T> Stream for TimeBinnedStream<T>
where
T: TimeBinnable + Unpin,
T: TimeBinnableTy + Unpin,
{
type Item = Sitemty<<<T as TimeBinnable>::TimeBinner as TimeBinner>::Output>;
type Item = Sitemty<<<T as TimeBinnableTy>::TimeBinner as TimeBinnerTy>::Output>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let span = tracing::span!(tracing::Level::TRACE, "poll");
let span = span!(Level::INFO, "poll");
let _spg = span.enter();
loop {
break if self.complete {

29
streams/src/timebinnedjson.rs
View File

@@ -6,6 +6,7 @@ use futures_util::stream;
use futures_util::StreamExt;
use items_0::streamitem::sitem_data;
use items_0::streamitem::Sitemty;
use items_0::timebin::TimeBinned;
use items_2::channelevents::ChannelEvents;
use items_2::merger::Merger;
use netpod::log::*;
@@ -20,29 +21,41 @@ use std::time::Instant;
pub async fn timebinned_json(query: &BinnedQuery, chconf: &ChConf, cluster: &Cluster) -> Result<JsonValue, Error> {
let binned_range = BinnedRangeEnum::covering_range(query.range().clone(), query.bin_count())?;
let bins_max = 10000;
//let do_time_weight = query.agg_kind().do_time_weighted();
warn!("TODO add with_deadline to PlainEventsQuery");
let deadline = Instant::now() + query.timeout_value();
let empty = items_2::empty::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape)?;
error!("TODO feed through transform chain");
warn!("TODO feed through transform chain");
let empty = ChannelEvents::Events(empty);
let empty = sitem_data(empty);
error!("TODO add with_deadline to PlainEventsQuery");
todo!();
let evquery = PlainEventsQuery::new(query.channel().clone(), query.range().clone());
// TODO
let evquery = PlainEventsQuery::new(query.channel().clone(), query.range().clone()).for_time_weighted_scalar();
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&evquery, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
info!("timebinned_json with empty item {empty:?}");
let stream = Merger::new(inps, 1024);
let stream = stream::iter([empty]).chain(stream);
let stream = RangeFilter2::new(stream, todo!(), evquery.one_before_range());
// TODO
let do_time_weight = true;
let one_before_range = true;
// TODO RangeFilter2 must accept SeriesRange
let range = query.range().try_into()?;
let stream = RangeFilter2::new(stream, range, one_before_range);
let stream = Box::pin(stream);
let do_time_weight = todo!();
// TODO TimeBinnedStream must accept types bin edges.
// Maybe even take a BinnedRangeEnum?
let stream = TimeBinnedStream::new(stream, todo!(), do_time_weight, deadline);
if false {
let mut stream = stream;
let _: Option<Sitemty<Box<dyn items_0::TimeBinned>>> = stream.next().await;
let _: Option<Sitemty<Box<dyn TimeBinned>>> = stream.next().await;
panic!()
}
// TODO collect should not have to accept two ranges, instead, generalize over it.
let collected = crate::collect::collect(stream, deadline, bins_max, None, Some(binned_range.clone())).await?;
let jsval = serde_json::to_value(&collected)?;
Ok(jsval)