daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up, collect with timeout

Browse Source
This commit is contained in:
Dominik Werder
2022-12-19 14:09:37 +01:00
parent 64233b0ccb
commit 646ec38b3c
32 changed files with 622 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
streams/src/timebinnedjson.rs
View File

@@ -1,19 +1,30 @@
use crate::tcprawclient::open_tcp_streams;
use err::Error;
use futures_util::stream;
use futures_util::StreamExt;
use items_2::channelevents::ChannelEvents;
#[allow(unused)]
use netpod::log::*;
use netpod::query::{BinnedQuery, PlainEventsQuery};
use netpod::{BinnedRange, Cluster};
use netpod::query::BinnedQuery;
use netpod::query::PlainEventsQuery;
use netpod::BinnedRange;
use netpod::ChConf;
use netpod::Cluster;
use serde_json::Value as JsonValue;
use std::time::{Duration, Instant};
use std::time::Duration;
use std::time::Instant;
pub async fn timebinned_json(query: &BinnedQuery, cluster: &Cluster) -> Result<JsonValue, Error> {
pub async fn timebinned_json(query: &BinnedQuery, chconf: &ChConf, cluster: &Cluster) -> Result<JsonValue, Error> {
let binned_range = BinnedRange::covering_range(query.range().clone(), query.bin_count())?;
let events_max = 10000;
let bins_max = 10000;
let do_time_weight = query.agg_kind().do_time_weighted();
let timeout = Duration::from_millis(7500);
let deadline = Instant::now() + timeout;
let _empty = items::empty_events_dyn(&chconf.scalar_type, &chconf.shape, query.agg_kind());
let _empty = items_2::empty_events_dyn(&chconf.scalar_type, &chconf.shape, query.agg_kind());
let empty = items_2::empty_events_dyn_2(&chconf.scalar_type, &chconf.shape, query.agg_kind());
let empty = ChannelEvents::Events(empty);
let empty = items::sitem_data(empty);
let rawquery = PlainEventsQuery::new(
query.channel().clone(),
query.range().clone(),
@@ -25,6 +36,7 @@ pub async fn timebinned_json(query: &BinnedQuery, cluster: &Cluster) -> Result<J
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&rawquery, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
let stream = { items_2::merger::Merger::new(inps, 1) };
let stream = stream::iter([empty]).chain(stream);
let stream = Box::pin(stream);
let stream = crate::timebin::TimeBinnedStream::new(stream, binned_range.edges(), do_time_weight, deadline);
if false {
@@ -32,7 +44,7 @@ pub async fn timebinned_json(query: &BinnedQuery, cluster: &Cluster) -> Result<J
let _: Option<items::Sitemty<Box<dyn items_0::TimeBinned>>> = stream.next().await;
panic!()
}
let collected = crate::collect::collect(stream, deadline, events_max).await?;
let collected = crate::collect::collect(stream, deadline, bins_max, None, Some(binned_range.clone())).await?;
let jsval = serde_json::to_value(&collected)?;
Ok(jsval)
}