daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Item clone test

Browse Source
This commit is contained in:
Dominik Werder
2023-05-08 22:11:27 +02:00
parent 11047b6b03
commit 0040705b04
18 changed files with 390 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
streams/src/itemclone.rs Normal file
View File

@@ -0,0 +1,88 @@
use async_channel::Send;
use async_channel::Sender;
use err::Error;
use futures_util::pin_mut;
use futures_util::Future;
use futures_util::Stream;
use futures_util::StreamExt;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
#[pin_project::pin_project]
pub struct Itemclone<T, INP>
where
T: 'static,
{
#[pin]
sender: Sender<T>,
inp: INP,
send_fut: Option<Send<'static, T>>,
}
impl<T, INP> Itemclone<T, INP> {
pub fn new(inp: INP, sender: Sender<T>) -> Self
where
INP: Stream<Item = T> + Unpin,
T: Clone + Unpin,
{
Self {
sender,
inp,
send_fut: None,
}
}
}
impl<T, INP> Itemclone<T, INP>
where
INP: Stream<Item = T> + Unpin,
T: Clone + Unpin,
{
fn poll_fresh(&mut self, cx: &mut Context) -> Poll<Option<Result<T, Error>>> {
use Poll::*;
match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => {
let sender = unsafe { &mut *((&mut self.sender) as *mut Sender<T>) };
self.send_fut = Some(sender.send(item.clone()));
Ready(Some(Ok(item)))
}
Ready(None) => {
self.sender.close();
Ready(None)
}
Pending => Pending,
}
}
fn send_copy(fut: &mut Send<T>, cx: &mut Context) -> Poll<Result<(), Error>> {
use Poll::*;
pin_mut!(fut);
match fut.poll(cx) {
Ready(Ok(())) => Ready(Ok(())),
Ready(Err(e)) => Ready(Err(e.into())),
Pending => Pending,
}
}
}
impl<T, INP> Stream for Itemclone<T, INP>
where
INP: Stream<Item = T> + Unpin,
T: Clone + Unpin,
{
type Item = Result<T, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let proj = self.as_mut().project();
match self.send_fut.as_mut() {
Some(fut) => match Self::send_copy(fut, cx) {
Ready(Ok(())) => self.poll_fresh(cx),
Ready(Err(e)) => Ready(Some(Err(e))),
Pending => Pending,
},
None => self.poll_fresh(cx),
}
}
}

1
streams/src/lib.rs
View File

@@ -4,6 +4,7 @@ pub mod dtflags;
pub mod filechunkread;
pub mod frames;
pub mod generators;
pub mod itemclone;
pub mod needminbuffer;
pub mod plaineventsjson;
pub mod rangefilter2;

131
streams/src/test/timebin.rs
View File

@@ -1,6 +1,9 @@
use crate::collect::collect;
use crate::generators::GenerateI32V00;
use crate::generators::GenerateI32V01;
use crate::itemclone::Itemclone;
use crate::test::runfut;
use crate::timebin::TimeBinnedStream;
use crate::transform::build_event_transform;
use err::Error;
use futures_util::stream;
@@ -8,8 +11,11 @@ use futures_util::StreamExt;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::timebin::TimeBinnable;
use items_0::timebin::TimeBinned;
use items_0::AppendAllFrom;
use items_0::Empty;
use items_2::binsdim0::BinsDim0;
use items_2::channelevents::ChannelEvents;
@@ -70,7 +76,7 @@ fn time_bin_00() -> Result<(), Error> {
d.push_back(bins);
d
};
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true);
let mut binned_stream = TimeBinnedStream::new(stream0, binned_range, true);
while let Some(item) = binned_stream.next().await {
eprintln!("{item:?}");
match item {
@@ -135,7 +141,7 @@ fn time_bin_01() -> Result<(), Error> {
}
});
let stream0 = Box::pin(stream0);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true);
let mut binned_stream = TimeBinnedStream::new(stream0, binned_range, true);
while let Some(item) = binned_stream.next().await {
if true {
eprintln!("{item:?}");
@@ -199,7 +205,7 @@ fn time_bin_02() -> Result<(), Error> {
x
});
let stream = Box::pin(stream);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream, binned_range.clone(), do_time_weight);
let mut binned_stream = TimeBinnedStream::new(stream, binned_range.clone(), do_time_weight);
// From there on it should no longer be neccessary to distinguish whether its still events or time bins.
// Then, optionally collect for output type like json, or stream as batches.
// TODO the timebinner should already provide batches to make this efficient.
@@ -280,7 +286,7 @@ fn time_bin_03() -> Result<(), Error> {
sitem_data(v01),
sitem_data(v03),
]));
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true);
let mut binned_stream = TimeBinnedStream::new(stream0, binned_range, true);
while let Some(item) = binned_stream.next().await {
eprintln!("{item:?}");
match item {
@@ -312,3 +318,120 @@ fn transform_chain_correctness_00() -> Result<(), Error> {
build_event_transform(&tq)?;
Ok(())
}
#[test]
fn timebin_multi_stage_00() -> Result<(), Error> {
// TODO chain two timebin stages with different binning grid.
let fut = async {
let do_time_weight = true;
let one_before_range = do_time_weight;
let range = nano_range_from_str("1970-01-01T00:00:10Z", "1970-01-01T00:01:03Z")?;
let range = SeriesRange::TimeRange(range);
let stream_evs = GenerateI32V01::new(0, 1, range.clone(), one_before_range);
let mut exp1 = {
let mut bins = BinsDim0::<i32>::empty();
for i in 0..53 {
bins.push(
SEC * (10 + i),
SEC * (11 + i),
2,
20 + 2 * i as i32,
21 + 2 * i as i32,
20.5 + 2. * i as f32,
);
}
bins
};
// NOTE:
// can store all bins in cache for which there is some non-empty bin following, or if the container has range-final.
let (q1tx, q1rx) = async_channel::bounded(128);
let (q2tx, q2rx) = async_channel::bounded(128);
let stream_evs = Box::pin(stream_evs);
let binned_stream = {
let binned_range = BinnedRangeEnum::covering_range(range.clone(), 48)?;
dbg!(&binned_range);
TimeBinnedStream::new(stream_evs, binned_range, do_time_weight).map(|x| {
//eprintln!("STAGE 1 -- {:?}", x);
x
})
};
let binned_stream = Itemclone::new(binned_stream, q1tx).map(|x| match x {
Ok(x) => x,
Err(e) => Err(e),
});
let binned_stream = {
let binned_range = BinnedRangeEnum::covering_range(range.clone(), 22)?;
dbg!(&binned_range);
TimeBinnedStream::new(Box::pin(binned_stream), binned_range, do_time_weight).map(|x| {
eprintln!("STAGE -- 2 {:?}", x);
x
})
};
let binned_stream = Itemclone::new(binned_stream, q2tx).map(|x| match x {
Ok(x) => x,
Err(e) => Err(e),
});
let mut have_range_final = false;
let mut binned_stream = binned_stream;
while let Some(item) = binned_stream.next().await {
//eprintln!("{item:?}");
match item {
Ok(item) => match item {
StreamItem::DataItem(item) => match item {
RangeCompletableItem::Data(item) => {
if let Some(item) = item.as_any_ref().downcast_ref::<BinsDim0<i32>>() {
if false {
eprintln!("-----------------------");
eprintln!("item {:?}", item);
eprintln!("-----------------------");
}
} else {
return Err(Error::with_msg_no_trace(format!("bad, got item with unexpected type")));
}
}
RangeCompletableItem::RangeComplete => {
have_range_final = true;
}
},
StreamItem::Log(_) => {}
StreamItem::Stats(_) => {}
},
Err(e) => Err(e).unwrap(),
}
}
assert!(have_range_final);
{
eprintln!("---------------------------------------------------------------------");
let mut coll = BinsDim0::empty();
let stream = q1rx;
while let Ok(item) = stream.recv().await {
//eprintln!("RECV [q1rx] {:?}", item);
on_sitemty_data!(item, |mut item: Box<dyn TimeBinned>| {
if let Some(k) = item.as_any_mut().downcast_mut::<BinsDim0<i32>>() {
coll.append_all_from(k);
}
sitem_data(item)
});
}
eprintln!("collected 1: {:?}", coll);
assert_eq!(coll, exp1);
}
{
eprintln!("---------------------------------------------------------------------");
let mut coll = BinsDim0::empty();
let stream = q2rx;
while let Ok(item) = stream.recv().await {
//eprintln!("RECV [q2rx] {:?}", item);
on_sitemty_data!(item, |mut item: Box<dyn TimeBinned>| {
if let Some(k) = item.as_any_mut().downcast_mut::<BinsDim0<i32>>() {
coll.append_all_from(k);
}
sitem_data(item)
});
}
eprintln!("collected 1: {:?}", coll);
}
Ok(())
};
runfut(fut)
}