daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Can show pulse id diff from sf-databuffer

Browse Source
This commit is contained in:
Dominik Werder
2023-03-22 14:07:05 +01:00
parent d1c10e1712
commit e53b328f21
24 changed files with 421 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
streams/src/plaineventsjson.rs
View File

@@ -3,8 +3,11 @@ use crate::tcprawclient::open_tcp_streams;
use err::Error;
use futures_util::stream;
use futures_util::StreamExt;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use items_2::merger::Merger;
use netpod::log::*;
use netpod::ChConf;
use netpod::Cluster;
@@ -36,8 +39,13 @@ pub async fn plain_events_json(
info!("plain_events_json evquery {:?}", evquery);
//let ev_agg_kind = evquery.agg_kind().as_ref().map_or(AggKind::Plain, |x| x.clone());
//info!("plain_events_json ev_agg_kind {:?}", ev_agg_kind);
error!("TODO feed through transform chain");
let empty = items_2::empty::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape)?;
warn!("TODO feed through transform chain");
let empty = if query.transform().is_pulse_id_diff() {
use items_0::Empty;
Box::new(items_2::eventsdim0::EventsDim0::<i64>::empty())
} else {
items_2::empty::empty_events_dyn_ev(&chconf.scalar_type, &chconf.shape)?
};
info!("plain_events_json with empty item {}", empty.type_name());
let empty = ChannelEvents::Events(empty);
let empty = sitem_data(empty);
@@ -45,14 +53,51 @@ pub async fn plain_events_json(
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&evquery, cluster).await?;
//let inps = open_tcp_streams::<_, Box<dyn items_2::Events>>(&query, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
let stream = items_2::merger::Merger::new(inps, 1024);
let stream = Merger::new(inps, 1024);
// Transforms that keep state between batches of events, usually only useful after merge.
// Example: pulse-id-diff
use futures_util::Stream;
use items_0::streamitem::Sitemty;
use std::pin::Pin;
let stream: Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>> = if query.transform().is_pulse_id_diff() {
Box::pin(stream.map(|item| {
let mut pulse_last = None;
on_sitemty_data!(item, move |item| {
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
let x = match item {
ChannelEvents::Events(item) => {
let (tss, pulses) = items_0::EventsNonObj::into_tss_pulses(item);
let mut item = items_2::eventsdim0::EventsDim0::empty();
for (ts, pulse) in tss.into_iter().zip(pulses) {
let value = if let Some(last) = pulse_last {
pulse as i64 - last as i64
} else {
0
};
item.push(ts, pulse, value);
pulse_last = Some(pulse);
}
ChannelEvents::Events(Box::new(item))
}
ChannelEvents::Status(x) => ChannelEvents::Status(x),
};
Ok(StreamItem::DataItem(RangeCompletableItem::Data(x)))
})
}))
} else {
Box::pin(stream)
};
let stream = stream.map(|item| {
info!("item after merge: {item:?}");
//info!("item after merge: {item:?}");
item
});
let stream = RangeFilter2::new(stream, todo!(), evquery.one_before_range());
let stream = RangeFilter2::new(stream, query.range().try_into()?, evquery.one_before_range());
let stream = stream.map(|item| {
info!("item after rangefilter: {item:?}");
//info!("item after rangefilter: {item:?}");
item
});
let stream = stream::iter([empty]).chain(stream);

2
streams/src/timebinnedjson.rs
View File

@@ -32,7 +32,7 @@ pub async fn timebinned_json(query: &BinnedQuery, chconf: &ChConf, cluster: &Clu
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&evquery, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
info!("timebinned_json with empty item {empty:?}");
let stream = Merger::new(inps, 128);
let stream = Merger::new(inps, 1024);
let stream = stream::iter([empty]).chain(stream);
let stream = RangeFilter2::new(stream, todo!(), evquery.one_before_range());
let stream = Box::pin(stream);