daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2023-03-09 15:53:14 +01:00
parent f262f7e9df
commit 431d98ffea
19 changed files with 362 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
streams/src/collect.rs
View File

@@ -7,9 +7,9 @@ use items_0::streamitem::Sitemty;
use items_0::streamitem::StatsItem;
use items_0::streamitem::StreamItem;
use netpod::log::*;
use netpod::BinnedRange;
use netpod::BinnedRangeEnum;
use netpod::DiskStats;
use netpod::NanoRange;
use netpod::SeriesRange;
use std::fmt;
use std::time::Duration;
use std::time::Instant;
@@ -37,8 +37,8 @@ async fn collect_in_span<T, S>(
stream: S,
deadline: Instant,
events_max: u64,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
range: Option<SeriesRange>,
binrange: Option<BinnedRangeEnum>,
) -> Result<Box<dyn items_0::collect_c::Collected>, Error>
where
S: Stream<Item = Sitemty<T>> + Unpin,
@@ -136,8 +136,8 @@ pub async fn collect<T, S>(
stream: S,
deadline: Instant,
events_max: u64,
range: Option<NanoRange>,
binrange: Option<BinnedRange>,
range: Option<SeriesRange>,
binrange: Option<BinnedRangeEnum>,
) -> Result<Box<dyn items_0::collect_c::Collected>, Error>
where
S: Stream<Item = Sitemty<T>> + Unpin,

9
streams/src/plaineventsjson.rs
View File

@@ -35,9 +35,10 @@ pub async fn plain_events_json(
let events_max = query.events_max();
let evquery = query.clone();
info!("plain_events_json evquery {:?}", evquery);
let ev_agg_kind = evquery.agg_kind().as_ref().map_or(AggKind::Plain, |x| x.clone());
info!("plain_events_json ev_agg_kind {:?}", ev_agg_kind);
let empty = items_2::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape, &ev_agg_kind)?;
//let ev_agg_kind = evquery.agg_kind().as_ref().map_or(AggKind::Plain, |x| x.clone());
//info!("plain_events_json ev_agg_kind {:?}", ev_agg_kind);
error!("TODO feed through transform chain");
let empty = items_2::empty::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape)?;
info!("plain_events_json with empty item {}", empty.type_name());
let empty = ChannelEvents::Events(empty);
let empty = sitem_data(empty);
@@ -50,7 +51,7 @@ pub async fn plain_events_json(
info!("item after merge: {item:?}");
item
});
let stream = RangeFilter2::new(stream, query.range().clone(), evquery.one_before_range());
let stream = RangeFilter2::new(stream, todo!(), evquery.one_before_range());
let stream = stream.map(|item| {
info!("item after rangefilter: {item:?}");
item

2
streams/src/timebin.rs
View File

@@ -91,7 +91,7 @@ where
trace!("process_item {item:?}");
if self.binner.is_none() {
trace!("process_item call time_binner_new");
let binner = item.time_binner_new(self.edges.clone(), self.do_time_weight);
let binner = item.time_binner_new(todo!(), self.do_time_weight);
self.binner = Some(binner);
}
let binner = self.binner.as_mut().unwrap();

24
streams/src/timebinnedjson.rs
View File

@@ -12,35 +12,33 @@ use items_2::merger::Merger;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::query::PlainEventsQuery;
use netpod::BinnedRange;
use netpod::BinnedRangeEnum;
use netpod::ChConf;
use netpod::Cluster;
use serde_json::Value as JsonValue;
use std::time::Instant;
pub async fn timebinned_json(query: &BinnedQuery, chconf: &ChConf, cluster: &Cluster) -> Result<JsonValue, Error> {
let binned_range = BinnedRange::covering_range(query.range().clone(), query.bin_count())?;
let binned_range = BinnedRangeEnum::covering_range(query.range().clone(), query.bin_count())?;
let bins_max = 10000;
let do_time_weight = query.agg_kind().do_time_weighted();
//let do_time_weight = query.agg_kind().do_time_weighted();
let deadline = Instant::now() + query.timeout_value();
let empty = items_2::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape, &query.agg_kind())?;
let empty = items_2::empty::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape)?;
error!("TODO feed through transform chain");
let empty = ChannelEvents::Events(empty);
let empty = sitem_data(empty);
let evquery = PlainEventsQuery::new(
query.channel().clone(),
query.range().clone(),
Some(query.agg_kind()),
query.timeout(),
None,
);
error!("TODO add with_deadline to PlainEventsQuery");
todo!();
let evquery = PlainEventsQuery::new(query.channel().clone(), query.range().clone());
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&evquery, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
info!("timebinned_json with empty item {empty:?}");
let stream = Merger::new(inps, 128);
let stream = stream::iter([empty]).chain(stream);
let stream = RangeFilter2::new(stream, query.range().clone(), evquery.one_before_range());
let stream = RangeFilter2::new(stream, todo!(), evquery.one_before_range());
let stream = Box::pin(stream);
let stream = TimeBinnedStream::new(stream, binned_range.edges(), do_time_weight, deadline);
let do_time_weight = todo!();
let stream = TimeBinnedStream::new(stream, todo!(), do_time_weight, deadline);
if false {
let mut stream = stream;
let _: Option<Sitemty<Box<dyn items_0::TimeBinned>>> = stream.next().await;