Imagebuffer reads, time binning

This commit is contained in:
Dominik Werder
2023-01-19 20:05:25 +01:00
parent 9c68476626
commit 8495853f8e
26 changed files with 341 additions and 127 deletions

View File

@@ -57,7 +57,7 @@ where
break;
}
};
trace!("collect_in_span see item");
trace!("collect_in_span see item {item:?}");
match item {
Ok(item) => match item {
StreamItem::DataItem(item) => match item {

View File

@@ -36,6 +36,7 @@ pub async fn plain_events_json(
let stream = inp0.chain(inp1).chain(inp2);
stream
};
netpod::log::info!("plain_events_json with empty item {empty:?}");
let stream = { items_2::merger::Merger::new(inps, 1) };
let stream = stream::iter([empty]).chain(stream);
let collected = crate::collect::collect(stream, deadline, events_max, Some(query.range().clone()), None).await?;

View File

@@ -81,11 +81,14 @@ where
}
fn process_item(&mut self, mut item: T) -> () {
trace!("process_item {item:?}");
if self.binner.is_none() {
trace!("process_item call time_binner_new");
let binner = item.time_binner_new(self.edges.clone(), self.do_time_weight);
self.binner = Some(binner);
}
let binner = self.binner.as_mut().unwrap();
trace!("process_item call binner ingest");
binner.ingest(&mut item);
}
}
@@ -198,9 +201,17 @@ where
Ready(Some(Err(e)))
}
} else {
trace2!("no bins ready yet");
self.done_data = true;
continue;
if let Some(bins) = binner.empty() {
trace!("at end of stream, bin count zero, return {bins:?}");
self.done_data = true;
Ready(Some(sitem_data(bins)))
} else {
error!("at the end, no bins, can not get empty");
self.done_data = true;
let e = Error::with_msg_no_trace(format!("no bins"))
.add_public_msg(format!("unable to produce bins"));
Ready(Some(Err(e)))
}
}
} else {
trace2!("input stream finished, still no binner");

View File

@@ -35,7 +35,8 @@ pub async fn timebinned_json(query: &BinnedQuery, chconf: &ChConf, cluster: &Clu
);
let inps = open_tcp_streams::<_, items_2::channelevents::ChannelEvents>(&rawquery, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader:
let stream = { items_2::merger::Merger::new(inps, 128) };
netpod::log::info!("timebinned_json with empty item {empty:?}");
let stream = items_2::merger::Merger::new(inps, 128);
let stream = stream::iter([empty]).chain(stream);
let stream = Box::pin(stream);
let stream = crate::timebin::TimeBinnedStream::new(stream, binned_range.edges(), do_time_weight, deadline);