daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disentangle deps, typechecks

Browse Source
This commit is contained in:
Dominik Werder
2024-11-07 17:44:37 +01:00
parent 09463302ee
commit 8fd7e72796
13 changed files with 402 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crates/streams/Cargo.toml
View File

@@ -5,8 +5,6 @@ authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2021"
[dependencies]
#tokio = { version = "1.41", features = ["io-util", "net", "time", "sync", "fs"] }
#tokio-stream = "0.1.16"
futures-util = "0.3.15"
pin-project = "1.0.12"
serde = { version = "1.0", features = ["derive"] }
@@ -32,12 +30,13 @@ http = "1"
http-body = "1"
http-body-util = "0.1.0"
[dev-dependencies]
taskrun = { path = "../taskrun" }
#[dev-dependencies]
#taskrun = { path = "../taskrun" }
[features]
wasm_transform = ["wasmer"]
indev = []
tests-runtime = []
[patch.crates-io]
thiserror = { git = "https://github.com/dominikwerder/thiserror.git", branch = "cstm" }

2
crates/streams/src/frames/inmem.rs
View File

@@ -14,7 +14,7 @@ use netpod::ByteSize;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use tokio::io::AsyncRead;
// use tokio::io::AsyncRead;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "InMem")]

265
crates/streams/src/rangefilter2.rs
View File

@@ -1,3 +1,6 @@
#[cfg(feature = "tests-runtime")]
mod test;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::sitem_err_from_string;
@@ -17,11 +20,6 @@ use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
#[cfg(test)]
use items_0::Events;
#[cfg(test)]
use std::collections::VecDeque;
macro_rules! trace_emit { ($det:expr, $($arg:tt)*) => ( if false && $det { trace!($($arg)*); } ) }
#[derive(Debug, thiserror::Error)]
@@ -276,260 +274,3 @@ where
debug!("drop {:?}", self);
}
}
#[test]
fn test_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut item1 = EventsDim0::<f32>::empty();
item1.push_back(beg.ns() + 0 * ms, 0, 3.);
item1.push_back(beg.ns() + 1 * ms, 0, 3.1);
item1.push_back(beg.ns() + 2 * ms, 0, 3.2);
item1.push_back(beg.ns() + 3 * ms, 0, 3.3);
item1.push_back(beg.ns() + 4 * ms, 0, 3.4);
item1.push_back(end.ns() - 1, 0, 4.0);
item1.push_back(end.ns() + 0, 0, 4.1);
item1.push_back(end.ns() + 1, 0, 4.1);
let w1: Box<dyn Events> = Box::new(item1.clone());
let e1 = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w1)));
let inp = futures_util::stream::iter([e1]);
let one_before_range = false;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
]];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_cut_before_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 0 * ms, 0, 3.);
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = false;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 0 * ms, 0, 3.);
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
//
beg.ns() - 1,
],
&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_01() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
item.push_back(beg.ns() + 0 * ms, 0, 3.);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
// &[],
&[
//
beg.ns() - 1,
beg.ns() + 0 * ms,
],
&[
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_only() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let _ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
//
beg.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[cfg(test)]
async fn fetch_into_tss_items<INP>(mut inp: INP) -> VecDeque<VecDeque<u64>>
where
INP: Stream<Item = Sitemty<Box<dyn Events>>> + Unpin,
{
let mut tss_items = VecDeque::new();
while let Some(e) = inp.next().await {
if let Ok(StreamItem::DataItem(RangeCompletableItem::Data(evs))) = e {
eprintln!("{:?}", evs);
tss_items.push_back(Events::tss(&evs).clone());
} else {
eprintln!("other item ----------: {:?}", e);
}
}
tss_items
}

267
crates/streams/src/rangefilter2/test.rs Normal file
View File

@@ -0,0 +1,267 @@
use crate::rangefilter2::RangeFilter2;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Events;
use netpod::range::evrange::NanoRange;
use netpod::TsNano;
use std::collections::VecDeque;
#[test]
fn test_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut item1 = EventsDim0::<f32>::empty();
item1.push_back(beg.ns() + 0 * ms, 0, 3.);
item1.push_back(beg.ns() + 1 * ms, 0, 3.1);
item1.push_back(beg.ns() + 2 * ms, 0, 3.2);
item1.push_back(beg.ns() + 3 * ms, 0, 3.3);
item1.push_back(beg.ns() + 4 * ms, 0, 3.4);
item1.push_back(end.ns() - 1, 0, 4.0);
item1.push_back(end.ns() + 0, 0, 4.1);
item1.push_back(end.ns() + 1, 0, 4.1);
let w1: Box<dyn Events> = Box::new(item1.clone());
let e1 = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w1)));
let inp = futures_util::stream::iter([e1]);
let one_before_range = false;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
]];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_cut_before_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 0 * ms, 0, 3.);
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = false;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_00() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 0 * ms, 0, 3.);
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
//
beg.ns() - 1,
],
&[
beg.ns() + 0 * ms,
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_01() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
item.push_back(beg.ns() + 0 * ms, 0, 3.);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() + 1 * ms, 0, 3.1);
item.push_back(beg.ns() + 2 * ms, 0, 3.2);
item.push_back(beg.ns() + 3 * ms, 0, 3.3);
item.push_back(beg.ns() + 4 * ms, 0, 3.4);
item.push_back(end.ns() - 1, 0, 4.0);
item.push_back(end.ns() + 0, 0, 4.1);
item.push_back(end.ns() + 1, 0, 4.1);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
// &[],
&[
//
beg.ns() - 1,
beg.ns() + 0 * ms,
],
&[
beg.ns() + 1 * ms,
beg.ns() + 2 * ms,
beg.ns() + 3 * ms,
beg.ns() + 4 * ms,
end.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[test]
fn test_one_before_only() {
use items_0::Empty;
use items_2::eventsdim0::EventsDim0;
let _ms = 1_000_000;
let beg = TsNano::from_ms(1000 * 10);
let end = TsNano::from_ms(1000 * 20);
let mut items = Vec::new();
{
let mut item = EventsDim0::<f32>::empty();
item.push_back(beg.ns() - 1, 0, 2.9);
let w: Box<dyn Events> = Box::new(item.clone());
let e: Sitemty<_> = Ok(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(e);
}
let inp = futures_util::stream::iter(items);
let one_before_range = true;
let range = NanoRange::from((beg.ns(), end.ns()));
let stream = RangeFilter2::new(inp, range, one_before_range);
let fut = async move {
let tss_items = fetch_into_tss_items(stream).await;
let exp: &[&[u64]] = &[
// TODO in the future this empty may be discarded
&[],
&[
//
beg.ns() - 1,
],
];
assert_eq!(&tss_items, &exp);
Ok::<_, Error>(())
};
taskrun::run(fut).unwrap();
}
#[cfg(test)]
async fn fetch_into_tss_items<INP>(mut inp: INP) -> VecDeque<VecDeque<u64>>
where
INP: Stream<Item = Sitemty<Box<dyn Events>>> + Unpin,
{
let mut tss_items = VecDeque::new();
while let Some(e) = inp.next().await {
if let Ok(StreamItem::DataItem(RangeCompletableItem::Data(evs))) = e {
eprintln!("{:?}", evs);
tss_items.push_back(Events::tss(&evs).clone());
} else {
eprintln!("other item ----------: {:?}", e);
}
}
tss_items
}

7
crates/streams/src/test.rs
View File

@@ -1,8 +1,5 @@
#[cfg(test)]
mod collect;
#[cfg(test)]
mod events;
#[cfg(test)]
mod timebin;
use futures_util::stream;
@@ -59,5 +56,7 @@ where
F: std::future::Future<Output = Result<T, E>>,
E: std::error::Error,
{
taskrun::run(fut)
// taskrun::run(fut)
let _ = fut;
todo!()
}