WIP on collector
This commit is contained in:
Dominik Werder
@@ -1,106 +1,120 @@
|
||||
use err::Error;
|
||||
use futures_util::{Stream, StreamExt};
|
||||
use items::{RangeCompletableItem, Sitemty, StreamItem};
|
||||
use items_2::streams::{Collectable, Collector};
|
||||
use items_2::collect::{Collectable, Collector};
|
||||
use netpod::log::*;
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
// This is meant to work with trait object event containers (crate items_2)
|
||||
#[allow(unused)]
|
||||
macro_rules! trace2 {
|
||||
(D$($arg:tt)*) => ();
|
||||
($($arg:tt)*) => (eprintln!($($arg)*));
|
||||
}
|
||||
|
||||
// TODO rename, it is also used for binned:
|
||||
pub async fn collect_plain_events_json<T, S>(stream: S, timeout: Duration, events_max: u64) -> Result<JsonValue, Error>
|
||||
#[allow(unused)]
|
||||
macro_rules! trace3 {
|
||||
(D$($arg:tt)*) => ();
|
||||
($($arg:tt)*) => (eprintln!($($arg)*));
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
macro_rules! trace4 {
|
||||
(D$($arg:tt)*) => ();
|
||||
($($arg:tt)*) => (eprintln!($($arg)*));
|
||||
}
|
||||
|
||||
pub async fn collect<T, S>(
|
||||
stream: S,
|
||||
deadline: Instant,
|
||||
events_max: u64,
|
||||
) -> Result<<<T as Collectable>::Collector as Collector>::Output, Error>
|
||||
where
|
||||
S: Stream<Item = Sitemty<T>> + Unpin,
|
||||
T: Collectable + fmt::Debug,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
// TODO in general a Collector does not need to know about the expected number of bins.
|
||||
// It would make more sense for some specific Collector kind to know.
|
||||
// Therefore introduce finer grained types.
|
||||
let mut collector: Option<Box<dyn Collector>> = None;
|
||||
let mut i1 = 0;
|
||||
let mut collector: Option<<T as Collectable>::Collector> = None;
|
||||
let mut stream = stream;
|
||||
let deadline = deadline.into();
|
||||
let mut range_complete = false;
|
||||
let mut total_duration = Duration::ZERO;
|
||||
loop {
|
||||
let item = if i1 == 0 {
|
||||
stream.next().await
|
||||
} else {
|
||||
if false {
|
||||
None
|
||||
} else {
|
||||
match tokio::time::timeout_at(deadline, stream.next()).await {
|
||||
Ok(k) => k,
|
||||
Err(_) => {
|
||||
eprintln!("TODO [smc3j3rwha732ru8wcnfgi]");
|
||||
err::todo();
|
||||
//collector.set_timed_out();
|
||||
None
|
||||
}
|
||||
let item = match tokio::time::timeout_at(deadline, stream.next()).await {
|
||||
Ok(Some(k)) => k,
|
||||
Ok(None) => break,
|
||||
Err(_e) => {
|
||||
if let Some(coll) = collector.as_mut() {
|
||||
coll.set_timed_out();
|
||||
} else {
|
||||
eprintln!("TODO [861a95813]");
|
||||
err::todo();
|
||||
}
|
||||
break;
|
||||
}
|
||||
};
|
||||
match item {
|
||||
Some(item) => {
|
||||
match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::Log(item) => {
|
||||
trace!("collect_plain_events_json log {:?}", item);
|
||||
Ok(item) => match item {
|
||||
StreamItem::DataItem(item) => match item {
|
||||
RangeCompletableItem::RangeComplete => {
|
||||
range_complete = true;
|
||||
if let Some(coll) = collector.as_mut() {
|
||||
coll.set_range_complete();
|
||||
} else {
|
||||
eprintln!("TODO [7cc0fca8f]");
|
||||
err::todo();
|
||||
}
|
||||
StreamItem::Stats(item) => {
|
||||
use items::StatsItem;
|
||||
use netpod::DiskStats;
|
||||
match item {
|
||||
// TODO factor and simplify the stats collection:
|
||||
StatsItem::EventDataReadStats(_) => {}
|
||||
StatsItem::RangeFilterStats(_) => {}
|
||||
StatsItem::DiskStats(item) => match item {
|
||||
DiskStats::OpenStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
DiskStats::SeekStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
DiskStats::ReadStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
DiskStats::ReadExactStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
RangeCompletableItem::Data(mut item) => {
|
||||
eprintln!("COLLECTOR INGEST ITEM");
|
||||
if collector.is_none() {
|
||||
let c = item.new_collector();
|
||||
collector = Some(c);
|
||||
}
|
||||
StreamItem::DataItem(item) => match item {
|
||||
RangeCompletableItem::RangeComplete => {
|
||||
eprintln!("TODO [73jdfcgf947d]");
|
||||
err::todo();
|
||||
//collector.set_range_complete();
|
||||
let coll = collector.as_mut().unwrap();
|
||||
coll.ingest(&mut item);
|
||||
if coll.len() as u64 >= events_max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
StreamItem::Log(item) => {
|
||||
trace!("Log {:?}", item);
|
||||
}
|
||||
StreamItem::Stats(item) => {
|
||||
trace!("Stats {:?}", item);
|
||||
use items::StatsItem;
|
||||
use netpod::DiskStats;
|
||||
match item {
|
||||
// TODO factor and simplify the stats collection:
|
||||
StatsItem::EventDataReadStats(_) => {}
|
||||
StatsItem::RangeFilterStats(_) => {}
|
||||
StatsItem::DiskStats(item) => match item {
|
||||
DiskStats::OpenStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
RangeCompletableItem::Data(item) => {
|
||||
eprintln!("TODO [nx298nu98venusfc8]");
|
||||
err::todo();
|
||||
//collector.ingest(&item);
|
||||
i1 += 1;
|
||||
if i1 >= events_max {
|
||||
break;
|
||||
}
|
||||
DiskStats::SeekStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
DiskStats::ReadStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
DiskStats::ReadExactStats(k) => {
|
||||
total_duration += k.duration;
|
||||
}
|
||||
},
|
||||
},
|
||||
Err(e) => {
|
||||
// TODO Need to use some flags to get good enough error message for remote user.
|
||||
Err(e)?;
|
||||
}
|
||||
};
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
// TODO Need to use some flags to get good enough error message for remote user.
|
||||
Err(e)?;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let _ = range_complete;
|
||||
let res = collector
|
||||
.ok_or_else(|| Error::with_msg_no_trace(format!("no collector created")))?
|
||||
.result()?;
|
||||
let ret = serde_json::to_value(&res)?;
|
||||
debug!("Total duration: {:?}", total_duration);
|
||||
Ok(ret)
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
#[cfg(test)]
|
||||
mod collect;
|
||||
#[cfg(test)]
|
||||
mod timebin;
|
||||
|
||||
use err::Error;
|
||||
use futures_util::{stream, Stream, StreamExt};
|
||||
use futures_util::{stream, Stream};
|
||||
use items::{sitem_data, Sitemty};
|
||||
use items_2::eventsdim0::EventsDim0;
|
||||
use items_2::merger_cev::ChannelEventsMerger;
|
||||
use items_2::{ChannelEvents, Empty};
|
||||
use netpod::timeunits::SEC;
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
type BoxedEventStream = Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>;
|
||||
|
||||
@@ -41,24 +41,6 @@ fn empty_input() -> Result<(), Error> {
|
||||
Err(Error::with_msg_no_trace("TODO"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_channel_events() -> Result<(), Error> {
|
||||
let fut = async {
|
||||
let inp0 = inmem_test_events_d0_i32_00();
|
||||
let inp1 = inmem_test_events_d0_i32_01();
|
||||
let mut merged = ChannelEventsMerger::new(vec![inp0, inp1]);
|
||||
while let Some(item) = merged.next().await {
|
||||
eprintln!("item {item:?}");
|
||||
}
|
||||
let timeout = Duration::from_millis(4000);
|
||||
let events_max = 10000;
|
||||
// TODO add event collection
|
||||
let collected = crate::collect::collect_plain_events_json(merged, timeout, events_max).await?;
|
||||
Ok(())
|
||||
};
|
||||
runfut(fut)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_mergeable_00() -> Result<(), Error> {
|
||||
let fut = async {
|
||||
|
||||
22
streams/src/test/collect.rs
Normal file
22
streams/src/test/collect.rs
Normal file
@@ -0,0 +1,22 @@
|
||||
use crate::test::runfut;
|
||||
use err::Error;
|
||||
use futures_util::stream;
|
||||
use items::sitem_data;
|
||||
use items_2::testgen::make_some_boxed_d0_f32;
|
||||
use netpod::timeunits::SEC;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[test]
|
||||
fn collect_channel_events() -> Result<(), Error> {
|
||||
let fut = async {
|
||||
let evs0 = make_some_boxed_d0_f32(20, SEC * 10, SEC * 1, 0, 28736487);
|
||||
let evs1 = make_some_boxed_d0_f32(20, SEC * 30, SEC * 1, 0, 882716583);
|
||||
let stream = stream::iter(vec![sitem_data(evs0), sitem_data(evs1)]);
|
||||
let deadline = Instant::now() + Duration::from_millis(4000);
|
||||
let events_max = 10000;
|
||||
let res = crate::collect::collect(stream, deadline, events_max).await?;
|
||||
eprintln!("collected result: {res:?}");
|
||||
Ok(())
|
||||
};
|
||||
runfut(fut)
|
||||
}
|
||||
Reference in New Issue
Block a user