daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-09-26 10:01:20 +02:00
parent ce797d2025
commit 633eee5a95
11 changed files with 219 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
crates/streams/src/collect.rs
View File

@@ -41,6 +41,11 @@ macro_rules! trace4 {
($($arg:tt)*) => (eprintln!($($arg)*));
}
pub enum CollectResult<T> {
Timeout,
Some(T),
}
pub struct Collect {
inp: Pin<Box<dyn Stream<Item = Sitemty<Box<dyn Collectable>>> + Send>>,
events_max: u64,
@@ -156,7 +161,7 @@ impl Collect {
}
impl Future for Collect {
type Output = Result<Box<dyn Collected>, Error>;
type Output = Result<CollectResult<Box<dyn Collected>>, Error>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
use Poll::*;
@@ -177,14 +182,13 @@ impl Future for Collect {
Some(mut coll) => match coll.result(self.range.clone(), self.binrange.clone()) {
Ok(res) => {
//info!("collect stats total duration: {:?}", total_duration);
Ready(Ok(res))
Ready(Ok(CollectResult::Some(res)))
}
Err(e) => Ready(Err(e)),
},
None => {
let e = Error::with_msg_no_trace(format!("no result because no collector was created"));
error!("{e}");
Ready(Err(e))
debug!("no result because no collector was created");
Ready(Ok(CollectResult::Timeout))
}
}
} else {
@@ -245,7 +249,7 @@ where
info!("collect_in_span call set_timed_out");
coll.set_timed_out();
} else {
warn!("collect timeout but no collector yet");
warn!("collect_in_span collect timeout but no collector yet");
}
break;
}
@@ -258,11 +262,11 @@ where
if let Some(coll) = collector.as_mut() {
coll.set_range_complete();
} else {
warn!("collect received RangeComplete but no collector yet");
warn!("collect_in_span received RangeComplete but no collector yet");
}
}
RangeCompletableItem::Data(mut item) => {
trace!("collect sees len {}", item.len());
trace!("collect_in_span sees len {}", item.len());
if collector.is_none() {
let c = item.new_collector();
collector = Some(c);
@@ -278,10 +282,10 @@ where
}
},
StreamItem::Log(item) => {
trace!("collect log {:?}", item);
trace!("collect_in_span log {:?}", item);
}
StreamItem::Stats(item) => {
trace!("collect stats {:?}", item);
trace!("collect_in_span stats {:?}", item);
match item {
// TODO factor and simplify the stats collection:
StatsItem::EventDataReadStats(_) => {}
@@ -313,9 +317,9 @@ where
let _ = range_complete;
let _ = timed_out;
let res = collector
.ok_or_else(|| Error::with_msg_no_trace(format!("no result because no collector was created")))?
.ok_or_else(|| Error::with_msg_no_trace(format!("no result, no collector created")))?
.result(range, binrange)?;
info!("collect stats total duration: {:?}", total_duration);
info!("collect_in_span stats total duration: {:?}", total_duration);
Ok(res)
}

14
crates/streams/src/plaineventsjson.rs
View File

@@ -1,4 +1,5 @@
use crate::collect::Collect;
use crate::collect::CollectResult;
use crate::firsterr::non_empty;
use crate::firsterr::only_first_err;
use crate::json_stream::events_stream_to_json_stream;
@@ -34,7 +35,7 @@ pub async fn plain_events_json(
ctx: &ReqCtx,
_cluster: &Cluster,
open_bytes: OpenBoxedBytesStreamsBox,
) -> Result<JsonValue, Error> {
) -> Result<CollectResult<JsonValue>, Error> {
debug!("plain_events_json evquery {:?}", evq);
let deadline = Instant::now() + evq.timeout().unwrap_or(Duration::from_millis(4000));
@@ -93,9 +94,14 @@ pub async fn plain_events_json(
.await
.map_err(Error::Collect)?;
debug!("plain_events_json collected");
let jsval = serde_json::to_value(&collected)?;
debug!("plain_events_json json serialized");
Ok(jsval)
if let CollectResult::Some(x) = collected {
let jsval = serde_json::to_value(&x)?;
debug!("plain_events_json json serialized");
Ok(CollectResult::Some(jsval))
} else {
debug!("plain_events_json timeout");
Ok(CollectResult::Timeout)
}
}
pub async fn plain_events_json_stream(

31
crates/streams/src/test/collect.rs
View File

@@ -1,4 +1,5 @@
use crate::collect::Collect;
use crate::collect::CollectResult;
use crate::test::runfut;
use crate::transform::build_event_transform;
use crate::transform::build_time_binning_transform;
@@ -11,8 +12,6 @@ use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::StreamItem;
use items_0::transform::EventStreamBox;
use items_0::transform::EventStreamTrait;
use items_0::WithLen;
use items_2::eventsdim0::EventsDim0CollectorOutput;
use items_2::streams::PlainEventStream;
@@ -69,14 +68,18 @@ fn collect_channel_events_01() -> Result<(), Error> {
let stream = TimeBinnableToCollectable::new(stream);
let stream = Box::pin(stream);
let res = Collect::new(stream, deadline, events_max, bytes_max, None, None).await?;
if let Some(res) = res.as_any_ref().downcast_ref::<EventsDim0CollectorOutput<f32>>() {
eprintln!("Great, a match");
eprintln!("{res:?}");
assert_eq!(res.len(), 40);
if let CollectResult::Some(res) = res {
if let Some(res) = res.as_any_ref().downcast_ref::<EventsDim0CollectorOutput<f32>>() {
eprintln!("Great, a match");
eprintln!("{res:?}");
assert_eq!(res.len(), 40);
} else {
return Err(Error::with_msg(format!("bad type of collected result")));
}
Ok(())
} else {
return Err(Error::with_msg(format!("bad type of collected result")));
}
Ok(())
};
runfut(fut)
}
@@ -110,14 +113,18 @@ fn collect_channel_events_pulse_id_diff() -> Result<(), Error> {
let stream = TimeBinnableToCollectable::new(stream);
let stream = Box::pin(stream);
let res = Collect::new(stream, deadline, events_max, bytes_max, None, None).await?;
if let Some(res) = res.as_any_ref().downcast_ref::<EventsDim0CollectorOutput<i64>>() {
eprintln!("Great, a match");
eprintln!("{res:?}");
assert_eq!(res.len(), 40);
if let CollectResult::Some(res) = res {
if let Some(res) = res.as_any_ref().downcast_ref::<EventsDim0CollectorOutput<i64>>() {
eprintln!("Great, a match");
eprintln!("{res:?}");
assert_eq!(res.len(), 40);
} else {
return Err(Error::with_msg(format!("bad type of collected result")));
}
Ok(())
} else {
return Err(Error::with_msg(format!("bad type of collected result")));
}
Ok(())
};
runfut(fut)
}

36
crates/streams/src/timebinnedjson.rs
View File

@@ -1,4 +1,5 @@
use crate::collect::Collect;
use crate::collect::CollectResult;
use crate::json_stream::JsonBytes;
use crate::json_stream::JsonStream;
use crate::rangefilter2::RangeFilter2;
@@ -316,7 +317,7 @@ async fn timebinned_stream(
events_read_provider: Option<Arc<dyn EventsReadProvider>>,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>, Error> {
use netpod::query::CacheUsage;
let cache_usage = query.cache_usage().unwrap_or(CacheUsage::Use);
let cache_usage = query.cache_usage().unwrap_or(CacheUsage::V0NoCache);
match (
ch_conf.series(),
cache_usage.clone(),
@@ -413,7 +414,7 @@ pub async fn timebinned_json(
open_bytes: OpenBoxedBytesStreamsBox,
cache_read_provider: Option<Arc<dyn CacheReadProvider>>,
events_read_provider: Option<Arc<dyn EventsReadProvider>>,
) -> Result<JsonValue, Error> {
) -> Result<CollectResult<JsonValue>, Error> {
let deadline = Instant::now()
+ query
.timeout_content()
@@ -438,19 +439,24 @@ pub async fn timebinned_json(
let collected = Collect::new(stream, deadline, collect_max, bytes_max, None, Some(binned_range));
let collected: BoxFuture<_> = Box::pin(collected);
let collres = collected.await?;
info!("timebinned_json collected type_name {:?}", collres.type_name());
let collres = if let Some(bins) = collres
.as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{
warn!("unexpected binned enum");
// bins.boxed_collected_with_enum_fix()
collres
} else {
collres
};
let jsval = serde_json::to_value(&collres)?;
Ok(jsval)
match collres {
CollectResult::Some(collres) => {
let collres = if let Some(bins) = collres
.as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{
debug!("unexpected binned enum");
// bins.boxed_collected_with_enum_fix()
collres
} else {
debug!("timebinned_json collected type_name {:?}", collres.type_name());
collres
};
let jsval = serde_json::to_value(&collres)?;
Ok(CollectResult::Some(jsval))
}
CollectResult::Timeout => Ok(CollectResult::Timeout),
}
}
fn take_collector_result(coll: &mut Box<dyn items_0::collect_s::Collector>) -> Option<serde_json::Value> {