daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-11-05 14:27:41 +01:00
parent 35d15c4694
commit ef021ff971
31 changed files with 266 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/streams/src/cbor_stream.rs
View File

@@ -18,7 +18,6 @@ use netpod::log::Level;
use netpod::log::*;
use netpod::ScalarType;
use netpod::Shape;
use std::fmt;
use std::io::Cursor;
use std::pin::Pin;
use std::task::Context;

7
crates/streams/src/frames/eventsfromframes.rs
View File

@@ -3,6 +3,7 @@ use futures_util::StreamExt;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::sitem_err_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::SitemErrTy;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_2::frame::decode_frame;
@@ -19,7 +20,7 @@ use std::task::Poll;
pub enum Error {}
pub struct EventsFromFrames<O> {
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, SitemErrTy>> + Send>>,
dbgdesc: String,
errored: bool,
completed: bool,
@@ -28,7 +29,7 @@ pub struct EventsFromFrames<O> {
impl<O> EventsFromFrames<O> {
pub fn new(
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, SitemErrTy>> + Send>>,
dbgdesc: String,
) -> Self {
Self {
@@ -98,7 +99,7 @@ where
e
);
self.errored = true;
Ready(Some(Err(e)))
Ready(Some(sitem_err_from_string(e)))
}
},
},

21
crates/streams/src/generators.rs
View File

@@ -10,7 +10,6 @@ use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::SitemErrTy;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
@@ -37,6 +36,8 @@ use std::time::Duration;
pub enum Error {
UnsupportedIsEventBlobs,
Transform(#[from] crate::transform::Error),
Items2(#[from] items_2::Error),
BadChannelName,
}
pub fn make_test_channel_events_bytes_stream(
@@ -68,7 +69,7 @@ pub fn make_test_channel_events_bytes_stream(
});
let stream = stream
.map_err(sitem_err2_from_string)
.map(|x| x.make_frame_dyn().map(|x| x.freeze()));
.map(|x| x.make_frame_dyn().map(|x| x.freeze()).map_err(sitem_err2_from_string));
let ret = Box::pin(stream);
Ok(ret)
}
@@ -106,14 +107,10 @@ fn make_test_channel_events_stream_data_inner(
} else {
let na: Vec<_> = chn.split("-").collect();
if na.len() != 3 {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
} else {
if na[0] != "inmem" {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
} else {
let _range = subq.range().clone();
if na[1] == "d0" {
@@ -124,14 +121,10 @@ fn make_test_channel_events_stream_data_inner(
//generator::generate_f32(node_ix, node_count, range)
panic!()
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
}
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
}
}
}

2
crates/streams/src/itemclone.rs
View File

@@ -65,7 +65,7 @@ where
pin_mut!(fut);
match fut.poll(cx) {
Ready(Ok(())) => Ready(Ok(())),
Ready(Err(e)) => Ready(Err(e.into())),
Ready(Err(_)) => todo!("can not send copy"),
Pending => Pending,
}
}

25
crates/streams/src/json_stream.rs
View File

@@ -2,6 +2,7 @@ use crate::cbor_stream::SitemtyDynEventsStream;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Events;
use items_0::WithLen;
@@ -11,7 +12,23 @@ use std::time::Duration;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "JsonStream")]
pub enum Error {}
pub enum Error {
Json(#[from] serde_json::Error),
Msg(String),
}
pub struct ErrMsg<E>(pub E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
pub struct JsonBytes(String);
@@ -56,7 +73,7 @@ pub fn events_stream_to_json_stream(stream: SitemtyDynEventsStream) -> impl Stre
prepend.chain(stream)
}
fn map_events(x: Result<StreamItem<RangeCompletableItem<Box<dyn Events>>>, Error>) -> Result<JsonBytes, Error> {
fn map_events(x: Sitemty<Box<dyn Events>>) -> Result<JsonBytes, Error> {
match x {
Ok(x) => match x {
StreamItem::DataItem(x) => match x {
@@ -104,12 +121,12 @@ fn map_events(x: Result<StreamItem<RangeCompletableItem<Box<dyn Events>>>, Error
}
},
StreamItem::Log(item) => {
info!("{item:?}");
debug!("{item:?}");
let item = JsonBytes::new(String::new());
Ok(item)
}
StreamItem::Stats(item) => {
info!("{item:?}");
debug!("{item:?}");
let item = JsonBytes::new(String::new());
Ok(item)
}

10
crates/streams/src/needminbuffer.rs
View File

@@ -12,7 +12,7 @@ use std::task::Poll;
pub enum Error {}
pub struct NeedMinBuffer {
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>> + Send>>,
need_min: u32,
left: Option<FileChunkRead>,
buf_len_histo: HistoLog2,
@@ -21,9 +21,11 @@ pub struct NeedMinBuffer {
}
impl NeedMinBuffer {
pub fn new(inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, Error>> + Send>>) -> Self {
pub fn new(
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>> + Send>>,
) -> Self {
Self {
inp: inp,
inp,
need_min: 1,
left: None,
buf_len_histo: HistoLog2::new(8),
@@ -50,7 +52,7 @@ impl Drop for NeedMinBuffer {
}
impl Stream for NeedMinBuffer {
type Item = Result<FileChunkRead, Error>;
type Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;

1
crates/streams/src/plaineventsjson.rs
View File

@@ -24,6 +24,7 @@ use std::time::Instant;
pub enum Error {
Stream(#[from] crate::plaineventsstream::Error),
Json(#[from] serde_json::Error),
Collect(#[from] crate::collect::Error),
}
pub async fn plain_events_json(

3
crates/streams/src/plaineventsstream.rs
View File

@@ -20,6 +20,7 @@ use std::pin::Pin;
pub enum Error {
Netpod(#[from] netpod::NetpodError),
Transform(#[from] crate::transform::Error),
TcpRawClient(#[from] crate::tcprawclient::Error),
}
pub type DynEventsStream = Pin<Box<dyn Stream<Item = Sitemty<Box<dyn Events>>> + Send>>;
@@ -77,7 +78,7 @@ pub async fn dyn_events_stream(
});
if let Some(wasmname) = evq.test_do_wasm() {
let stream = transform_wasm(stream, wasmname, ctx).await?;
let stream = transform_wasm::<_, items_0::streamitem::SitemErrTy>(stream, wasmname, ctx).await?;
Ok(Box::pin(stream))
} else {
Ok(Box::pin(stream))

28
crates/streams/src/rangefilter2.rs
View File

@@ -1,6 +1,6 @@
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::sitem_err_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StatsItem;
@@ -22,13 +22,12 @@ use items_0::Events;
#[cfg(test)]
use std::collections::VecDeque;
#[allow(unused)]
macro_rules! trace_emit {
($det:expr, $($arg:tt)*) => {
if $det {
eprintln!($($arg)*);
}
};
macro_rules! trace_emit { ($det:expr, $($arg:tt)*) => ( if false && $det { trace!($($arg)*); } ) }
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Rangefilter")]
pub enum Error {
Merge(#[from] MergeError),
}
pub struct RangeFilter2<S, ITY>
@@ -146,8 +145,7 @@ where
} else {
trace_emit!(self.trdet, "discarding events len {:?}", lige - 1);
let mut dummy = item.new_empty();
item.drain_into(&mut dummy, (0, lige - 1))
.map_err(|e| format!("{e} unexpected MergeError while remove of items"))?;
item.drain_into(&mut dummy, (0, lige - 1))?;
self.slot1 = None;
item
}
@@ -157,8 +155,7 @@ where
trace_emit!(self.trdet, "drain into to keep one before");
let n = item.len();
let mut keep = item.new_empty();
item.drain_into(&mut keep, (n.max(1) - 1, n))
.map_err(|e| format!("{e} unexpected MergeError while remove of items"))?;
item.drain_into(&mut keep, (n.max(1) - 1, n))?;
self.slot1 = Some(keep);
item.new_empty()
}
@@ -192,10 +189,7 @@ where
loop {
break if self.complete {
error!("{} poll_next on complete", Self::type_name());
Ready(Some(Err(Error::with_msg_no_trace(format!(
"{} poll_next on complete",
Self::type_name()
)))))
Ready(Some(sitem_err_from_string("poll next on complete")))
} else if self.done {
self.complete = true;
Ready(None)
@@ -223,7 +217,7 @@ where
Err(e) => {
error!("sees: {e}");
self.inp_done = true;
Ready(Some(Err(e)))
Ready(Some(sitem_err_from_string(e)))
}
},
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)) => {

66
crates/streams/src/tcprawclient.rs
View File

@@ -7,14 +7,15 @@ use crate::frames::eventsfromframes::EventsFromFrames;
use crate::frames::inmem::BoxedBytesStream;
use crate::frames::inmem::InMemoryFrameStream;
use crate::frames::inmem::TcpReadAsBytes;
use err::Error;
use futures_util::Future;
use futures_util::Stream;
use futures_util::TryStreamExt;
use http::Uri;
use httpclient::body_bytes;
use httpclient::http;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::Sitemty;
use items_2::eventfull::EventFull;
use items_2::framable::EventQueryJsonStringFrame;
@@ -42,6 +43,40 @@ use tokio::net::TcpStream;
pub const TEST_BACKEND: &str = "testbackend-00";
#[derive(Debug, thiserror::Error)]
#[cstm(name = "TcpRawClient")]
pub enum Error {
IO(#[from] std::io::Error),
Msg(String),
Frame(#[from] items_2::frame::Error),
Framable(#[from] items_2::framable::Error),
Json(#[from] serde_json::Error),
Http(#[from] http::Error),
HttpClient(#[from] httpclient::Error),
Hyper(#[from] httpclient::hyper::Error),
#[error("ServerError({0:?}, {1})")]
ServerError(http::response::Parts, String),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
impl From<String> for Error {
fn from(value: String) -> Self {
Self::Msg(value)
}
}
pub trait OpenBoxedBytesStreams {
fn open(
&self,
@@ -75,8 +110,10 @@ pub async fn x_processed_event_blobs_stream_from_node_tcp(
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
let inp = Box::pin(TcpReadAsBytes::new(netin)) as BoxedBytesStream;
let inp = TcpReadAsBytes::new(netin).map_err(sitem_err2_from_string);
let inp = Box::pin(inp) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let items = EventsFromFrames::new(frames, addr);
Ok(Box::pin(items))
@@ -106,31 +143,20 @@ pub async fn x_processed_event_blobs_stream_from_node_http(
.header(header::HOST, uri.host().unwrap())
.header(header::ACCEPT, APP_OCTET)
.header(ctx.header_name(), ctx.header_value())
.body(body_bytes(buf))
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
.body(body_bytes(buf))?;
let mut client = httpclient::connect_client(req.uri()).await?;
let res = client
.send_request(req)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let res = client.send_request(req).await?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = httpclient::read_body_bytes(body).await?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
return Err(Error::ServerError(head, s.to_string()));
}
let (_head, body) = res.into_parts();
let inp = Box::pin(httpclient::IncomingStream::new(body)) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let stream = EventsFromFrames::new(frames, url.to_string());
debug!("open_event_data_streams_http done {url}");
@@ -175,8 +201,11 @@ where
netout.flush().await?;
netout.forget();
// TODO for images, we need larger buffer capacity
let inp = Box::pin(TcpReadAsBytes::new(netin)) as BoxedBytesStream;
let inp = TcpReadAsBytes::new(netin);
let inp = inp.map_err(sitem_err2_from_string);
let inp = Box::pin(inp) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
streams.push(Box::pin(stream) as _);
@@ -193,6 +222,7 @@ where
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
let frames = InMemoryFrameStream::new(inp, bufcap);
let frames = frames.map_err(sitem_err2_from_string);
// TODO let EventsFromFrames accept also non-boxed input?
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, dbgdesc);

12
crates/streams/src/test.rs
View File

@@ -5,7 +5,6 @@ mod events;
#[cfg(test)]
mod timebin;
use err::Error;
use futures_util::stream;
use futures_util::Stream;
use items_0::streamitem::sitem_data;
@@ -17,6 +16,10 @@ use items_2::eventsdim0::EventsDim0;
use netpod::timeunits::SEC;
use std::pin::Pin;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "StreamsTest")]
pub enum Error {}
type BoxedEventStream = Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>;
// TODO use some xorshift generator.
@@ -51,11 +54,10 @@ fn merge_mergeable_00() -> Result<(), Error> {
runfut(fut)
}
fn runfut<T, F>(fut: F) -> Result<T, err::Error>
fn runfut<F, T, E>(fut: F) -> Result<T, E>
where
F: std::future::Future<Output = Result<T, Error>>,
F: std::future::Future<Output = Result<T, E>>,
E: std::error::Error,
{
use futures_util::TryFutureExt;
let fut = fut.map_err(|e| e.into());
taskrun::run(fut)
}

5
crates/streams/src/test/events.rs
View File

@@ -8,6 +8,7 @@ use crate::tcprawclient::TEST_BACKEND;
use futures_util::future;
use futures_util::Future;
use futures_util::StreamExt;
use futures_util::TryFutureExt;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange;
@@ -83,8 +84,8 @@ impl OpenBoxedBytesStreams for StreamOpener {
&self,
subq: EventsSubQuery,
_ctx: ReqCtx,
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, Error>> + Send>> {
Box::pin(stream_opener(subq))
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, crate::tcprawclient::Error>> + Send>> {
Box::pin(stream_opener(subq).map_err(|e| crate::tcprawclient::Error::Msg(format!("{e}"))))
}
}

7
crates/streams/src/teststream.rs
View File

@@ -1,21 +1,20 @@
use crate::timebin::cached::reader::EventsReadProvider;
use crate::timebin::cached::reader::EventsReading;
use futures_util::Stream;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::Sitemty;
use items_2::channelevents::ChannelEvents;
use netpod::range::evrange::SeriesRange;
use query::api4::events::EventsSubQuery;
use rand_xoshiro::rand_core::SeedableRng;
use rand_xoshiro::Xoshiro128PlusPlus;
use std::pin::Pin;
fn make_stream(chname: &str, range: &SeriesRange) -> Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>> {
if chname == "unittest;scylla;cont;scalar;f32" {
let e = ::err::Error::with_msg_no_trace("unknown channel {chname}");
let e = sitem_err2_from_string(format!("unknown channel {chname}"));
let ret = futures_util::stream::iter([Err(e)]);
Box::pin(ret)
} else {
let e = ::err::Error::with_msg_no_trace("unknown channel {chname}");
let e = sitem_err2_from_string(format!("unknown channel {chname}"));
let ret = futures_util::stream::iter([Err(e)]);
Box::pin(ret)
}

20
crates/streams/src/timebinnedjson.rs
View File

@@ -14,6 +14,7 @@ use futures_util::Stream;
use futures_util::StreamExt;
use items_0::collect_s::CollectableDyn;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
@@ -41,6 +42,23 @@ pub enum Error {
Query(#[from] query::api4::binned::Error),
FromLayers(#[from] super::timebin::fromlayers::Error),
Transform(#[from] super::transform::Error),
TcpRawClient(#[from] crate::tcprawclient::Error),
Collect(#[from] crate::collect::Error),
Json(#[from] serde_json::Error),
Msg(String),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
#[allow(unused)]
@@ -435,7 +453,7 @@ pub async fn timebinned_json_framed(
// TODO skip the intermediate conversion to js value, go directly to string data
let stream = stream.map(|x| match x {
Ok(x) => Ok(JsonBytes::new(serde_json::to_string(&x).unwrap())),
Err(e) => Err(e),
Err(e) => Err(crate::json_stream::Error::from(crate::json_stream::ErrMsg(e))),
});
Ok(Box::pin(stream))
}

17
crates/streams/src/transform.rs
View File

@@ -20,23 +20,20 @@ use std::pin::Pin;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Transform")]
pub enum Error {}
pub enum Error {
#[error("UnhandledQuery({0:?})")]
UnhandledQuery(EventTransformQuery),
}
pub fn build_event_transform(tr: &TransformQuery) -> Result<TransformEvent, Error> {
let trev = tr.get_tr_event();
match trev {
EventTransformQuery::ValueFull => Ok(make_transform_identity()),
EventTransformQuery::MinMaxAvgDev => Ok(make_transform_min_max_avg()),
EventTransformQuery::ArrayPick(..) => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::ArrayPick(..) => Err(Error::UnhandledQuery(trev.clone())),
EventTransformQuery::PulseIdDiff => Ok(make_transform_pulse_id_diff()),
EventTransformQuery::EventBlobsVerbatim => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::EventBlobsUncompressed => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::EventBlobsVerbatim => Err(Error::UnhandledQuery(trev.clone())),
EventTransformQuery::EventBlobsUncompressed => Err(Error::UnhandledQuery(trev.clone())),
}
}