daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-11-05 14:27:41 +01:00
parent 35d15c4694
commit ef021ff971
31 changed files with 266 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/disk/src/eventchunker.rs
View File

@@ -140,7 +140,7 @@ impl EventChunker {
// TODO `expand` flag usage
pub fn from_start(
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>> + Send>>,
fetch_info: SfChFetchInfo,
range: NanoRange,
stats_conf: EventChunkerConf,

6
crates/disk/src/merge/mergedblobsfromremotes.rs
View File

@@ -1,7 +1,8 @@
use err::Error;
use futures_util::pin_mut;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryFutureExt;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::Sitemty;
use items_2::eventfull::EventFull;
use items_2::merger::Merger;
@@ -16,7 +17,7 @@ use std::task::Poll;
use streams::tcprawclient::x_processed_event_blobs_stream_from_node;
type T001<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
type T002<T> = Pin<Box<dyn Future<Output = Result<T001<T>, Error>> + Send>>;
type T002<T> = Pin<Box<dyn Future<Output = Result<T001<T>, items_0::streamitem::SitemErrTy>> + Send>>;
pub struct MergedBlobsFromRemotes {
tcp_establish_futs: Vec<T002<EventFull>>,
@@ -32,6 +33,7 @@ impl MergedBlobsFromRemotes {
let mut tcp_establish_futs = Vec::new();
for node in &cluster.nodes {
let f = x_processed_event_blobs_stream_from_node(subq.clone(), node.clone(), ctx.clone());
let f = f.map_err(sitem_err2_from_string);
let f: T002<EventFull> = Box::pin(f);
tcp_establish_futs.push(f);
}

4
crates/disk/src/raw/conn.rs
View File

@@ -73,7 +73,7 @@ pub async fn make_event_pipe(
128
};
let event_blobs = EventChunkerMultifile::new(
(&range).try_into()?,
(&range).try_into().map_err(Error::from_string)?,
fetch_info.clone(),
ncc.node.clone(),
ncc.ix,
@@ -129,7 +129,7 @@ pub fn make_event_blobs_pipe_real(
let range = subq.range();
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
let event_blobs = make_event_blobs_stream(
range.try_into()?,
range.try_into().map_err(Error::from_string)?,
fetch_info.clone(),
one_before,
event_chunker_conf,

6
crates/items_0/src/collect_s.rs
View File

@@ -20,7 +20,7 @@ pub trait ToJsonBytes {
}
pub trait ToJsonResult: fmt::Debug + AsAnyRef + AsAnyMut + Send {
fn to_json_value(&self) -> Result<serde_json::Value, Error>;
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error>;
}
impl AsAnyRef for serde_json::Value {
@@ -36,7 +36,7 @@ impl AsAnyMut for serde_json::Value {
}
impl ToJsonResult for serde_json::Value {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
Ok(self.clone())
}
}
@@ -50,7 +50,7 @@ impl ToJsonBytes for serde_json::Value {
pub trait CollectedDyn: fmt::Debug + TypeName + Send + AsAnyRef + WithLen + ToJsonResult {}
impl ToJsonResult for Box<dyn CollectedDyn> {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
ToJsonResult::to_json_value(self.as_ref())
}
}

5
crates/items_2/src/binning/container_bins.rs
View File

@@ -450,7 +450,7 @@ impl<EVT> ToJsonResult for ContainerBinsCollectorOutput<EVT>
where
EVT: EventValueType,
{
fn to_json_value(&self) -> Result<serde_json::Value, err::Error> {
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
let bins = &self.bins;
let ts1sns: Vec<_> = bins.ts1s.iter().map(|x| x.ns()).collect();
let ts2sns: Vec<_> = bins.ts2s.iter().map(|x| x.ns()).collect();
@@ -471,8 +471,7 @@ where
maxs,
avgs,
};
let ret = serde_json::to_value(&val).map_err(err::Error::from_string);
ret
serde_json::to_value(&val)
}
}

4
crates/items_2/src/binsdim0.rs
View File

@@ -731,8 +731,8 @@ impl<NTY> BinsDim0CollectedResult<NTY> {
}
impl<NTY: ScalarOps> ToJsonResult for BinsDim0CollectedResult<NTY> {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
serde_json::to_value(self).map_err(Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

4
crates/items_2/src/binsxbindim0.rs
View File

@@ -361,8 +361,8 @@ impl<NTY> BinsXbinDim0CollectedResult<NTY> {
}
impl<NTY: ScalarOps> ToJsonResult for BinsXbinDim0CollectedResult<NTY> {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
serde_json::to_value(self).map_err(Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

4
crates/items_2/src/channelevents.rs
View File

@@ -1023,8 +1023,8 @@ impl WithLen for ChannelEventsCollectorOutput {
}
impl items_0::collect_s::ToJsonResult for ChannelEventsCollectorOutput {
fn to_json_value(&self) -> Result<serde_json::Value, ::err::Error> {
serde_json::to_value(self).map_err(::err::Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

4
crates/items_2/src/eventsdim0.rs
View File

@@ -371,8 +371,8 @@ impl<STY: ScalarOps> WithLen for EventsDim0CollectorOutput<STY> {
}
impl<STY: ScalarOps> ToJsonResult for EventsDim0CollectorOutput<STY> {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
serde_json::to_value(self).map_err(Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

2
crates/items_2/src/eventsdim0enum.rs
View File

@@ -120,7 +120,7 @@ impl TypeName for EventsDim0EnumCollectorOutput {
}
impl ToJsonResult for EventsDim0EnumCollectorOutput {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
todo!()
}
}

4
crates/items_2/src/eventsdim1.rs
View File

@@ -334,8 +334,8 @@ impl<STY: ScalarOps> WithLen for EventsDim1CollectorOutput<STY> {
}
impl<STY: ScalarOps> ToJsonResult for EventsDim1CollectorOutput<STY> {
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
serde_json::to_value(self).map_err(Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

4
crates/items_2/src/eventsxbindim0.rs
View File

@@ -652,8 +652,8 @@ impl<NTY> ToJsonResult for EventsXbinDim0CollectorOutput<NTY>
where
NTY: ScalarOps,
{
fn to_json_value(&self) -> Result<serde_json::Value, Error> {
serde_json::to_value(self).map_err(Error::from_string)
fn to_json_value(&self) -> Result<serde_json::Value, serde_json::Error> {
serde_json::to_value(self)
}
}

38
crates/items_2/src/framable.rs
View File

@@ -4,7 +4,6 @@ use crate::frame::make_log_frame;
use crate::frame::make_range_complete_frame;
use crate::frame::make_stats_frame;
use bytes::BytesMut;
use err::Error;
use items_0::framable::FrameTypeInnerDyn;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::LogItem;
@@ -26,6 +25,27 @@ pub const INMEM_FRAME_HEAD: usize = 20;
pub const INMEM_FRAME_FOOT: usize = 4;
pub const INMEM_FRAME_MAGIC: u32 = 0xc6c3b73d;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "ItemFramable")]
pub enum Error {
Msg(String),
DummyError,
Frame(#[from] crate::frame::Error),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
pub trait FrameTypeStatic {
const FRAME_TYPE_ID: u32;
}
@@ -78,14 +98,16 @@ where
match self {
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))) => {
let frame_type_id = k.frame_type_id();
make_frame_2(self, frame_type_id)
make_frame_2(self, frame_type_id).map_err(Error::from)
}
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)) => make_range_complete_frame(),
Ok(StreamItem::Log(item)) => make_log_frame(item),
Ok(StreamItem::Stats(item)) => make_stats_frame(item),
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)) => {
make_range_complete_frame().map_err(Error::from)
}
Ok(StreamItem::Log(item)) => make_log_frame(item).map_err(Error::from),
Ok(StreamItem::Stats(item)) => make_stats_frame(item).map_err(Error::from),
Err(e) => {
info!("calling make_error_frame for [[{e}]]");
make_error_frame(e)
make_error_frame(e).map_err(Error::from)
}
}
}
@@ -186,7 +208,7 @@ fn test_frame_log() {
fn test_frame_error() {
use crate::channelevents::ChannelEvents;
use crate::frame::json_from_slice;
let item: Sitemty<ChannelEvents> = Err(Error::with_msg_no_trace(format!("dummy-error-message")));
let item: Sitemty<ChannelEvents> = items_0::streamitem::sitem_err_from_string("dummyerror");
let buf = Framable::make_frame_dyn(&item).unwrap();
let len = u32::from_le_bytes(buf[12..16].try_into().unwrap());
let tyid = u32::from_le_bytes(buf[8..12].try_into().unwrap());
@@ -194,5 +216,5 @@ fn test_frame_error() {
panic!("bad tyid");
}
eprintln!("buf len {} len {}", buf.len(), len);
let item2: Error = json_from_slice(&buf[20..20 + len as usize]).unwrap();
let item2: items_0::streamitem::SitemErrTy = json_from_slice(&buf[20..20 + len as usize]).unwrap();
}

76
crates/items_2/src/frame.rs
View File

@@ -13,7 +13,6 @@ use bincode::config::WithOtherTrailing;
use bincode::DefaultOptions;
use bytes::BufMut;
use bytes::BytesMut;
use err::Error;
use items_0::bincode;
use items_0::streamitem::LogItem;
use items_0::streamitem::StatsItem;
@@ -27,6 +26,37 @@ use serde::Serialize;
use std::any;
use std::io;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "ItemFrame")]
pub enum Error {
TooLongPayload(usize),
UnknownEncoder(u32),
#[error("BufferMismatch({0}, {1}, {2})")]
BufferMismatch(u32, usize, u32),
#[error("TyIdMismatch({0}, {1})")]
TyIdMismatch(u32, u32),
Msg(String),
Bincode(#[from] Box<bincode::ErrorKind>),
RmpEnc(#[from] rmp_serde::encode::Error),
RmpDec(#[from] rmp_serde::decode::Error),
ErasedSerde(#[from] erased_serde::Error),
Postcard(#[from] postcard::Error),
SerdeJson(#[from] serde_json::Error),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
pub fn bincode_ser<W>(
w: W,
) -> bincode::Serializer<
@@ -54,7 +84,7 @@ where
{
let mut out = Vec::new();
let mut ser = bincode_ser(&mut out);
item.serialize(&mut ser).map_err(|e| format!("{e}"))?;
item.serialize(&mut ser)?;
Ok(out)
}
@@ -68,14 +98,14 @@ where
.with_fixint_encoding()
.reject_trailing_bytes();
let mut de = bincode::Deserializer::from_slice(buf, opts);
<T as serde::Deserialize>::deserialize(&mut de).map_err(|e| format!("{e}").into())
<T as serde::Deserialize>::deserialize(&mut de).map_err(Into::into)
}
fn msgpack_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
where
T: Serialize,
{
rmp_serde::to_vec_named(&item).map_err(|e| format!("{e}").into())
rmp_serde::to_vec_named(&item).map_err(Error::from)
}
fn msgpack_erased_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
@@ -86,8 +116,7 @@ where
{
let mut ser1 = rmp_serde::Serializer::new(&mut out).with_struct_map();
let mut ser2 = <dyn erased_serde::Serializer>::erase(&mut ser1);
item.erased_serialize(&mut ser2)
.map_err(|e| Error::from(format!("{e}")))?;
item.erased_serialize(&mut ser2)?;
}
Ok(out)
}
@@ -96,14 +125,14 @@ fn msgpack_from_slice<T>(buf: &[u8]) -> Result<T, Error>
where
T: for<'de> serde::Deserialize<'de>,
{
rmp_serde::from_slice(buf).map_err(|e| format!("{e}").into())
rmp_serde::from_slice(buf).map_err(Error::from)
}
fn postcard_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
where
T: Serialize,
{
postcard::to_stdvec(&item).map_err(|e| format!("{e}").into())
postcard::to_stdvec(&item).map_err(Error::from)
}
fn postcard_erased_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
@@ -117,31 +146,30 @@ where
{
let mut ser2 = <dyn erased_serde::Serializer>::erase(&mut ser1);
item.erased_serialize(&mut ser2)
}
.map_err(|e| Error::from(format!("{e}")))?;
let ret = ser1.output.finalize().map_err(|e| format!("{e}").into());
ret
}?;
let ret = ser1.output.finalize()?;
Ok(ret)
}
pub fn postcard_from_slice<T>(buf: &[u8]) -> Result<T, Error>
where
T: for<'de> serde::Deserialize<'de>,
{
postcard::from_bytes(buf).map_err(|e| format!("{e}").into())
Ok(postcard::from_bytes(buf)?)
}
fn json_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
where
T: Serialize,
{
serde_json::to_vec(&item).map_err(Error::from_string)
Ok(serde_json::to_vec(&item)?)
}
pub fn json_from_slice<T>(buf: &[u8]) -> Result<T, Error>
where
T: for<'de> serde::Deserialize<'de>,
{
serde_json::from_slice(buf).map_err(Error::from_string)
Ok(serde_json::from_slice(buf)?)
}
pub fn encode_to_vec<T>(item: T) -> Result<Vec<u8>, Error>
@@ -187,7 +215,7 @@ where
{
let enc = encode_erased_to_vec(item)?;
if enc.len() > u32::MAX as usize {
return Err(Error::with_msg(format!("too long payload {}", enc.len())));
return Err(Error::TooLongPayload(enc.len()));
}
let mut h = crc32fast::Hasher::new();
h.update(&enc);
@@ -323,15 +351,10 @@ where
T: FrameDecodable,
{
if frame.encid() != INMEM_FRAME_ENCID {
return Err(Error::with_msg(format!("unknown encoder id {:?}", frame)));
return Err(Error::UnknownEncoder(frame.encid()));
}
if frame.len() as usize != frame.buf().len() {
return Err(Error::with_msg(format!(
"buf mismatch {} vs {} in {:?}",
frame.len(),
frame.buf().len(),
frame
)));
return Err(Error::BufferMismatch(frame.len(), frame.buf().len(), frame.tyid()));
}
if frame.tyid() == ERROR_FRAME_TYPE_ID {
// error frames are always encoded as json
@@ -376,12 +399,7 @@ where
} else {
let tyid = T::FRAME_TYPE_ID;
if frame.tyid() != tyid {
Err(Error::with_msg(format!(
"type id mismatch expect {:04x} found {:04x} {:?}",
tyid,
frame.tyid(),
frame
)))
Err(Error::TyIdMismatch(tyid, frame.tyid()))
} else {
match decode_from_slice(frame.buf()) {
Ok(item) => Ok(item),

9
crates/nodenet/src/client.rs
View File

@@ -1,5 +1,6 @@
use err::Error;
use futures_util::Future;
use futures_util::TryFutureExt;
use http::header;
use http::Method;
use http::Request;
@@ -8,6 +9,7 @@ use httpclient::http;
use httpclient::hyper::StatusCode;
use httpclient::hyper::Uri;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_2::framable::Framable;
use netpod::log::*;
use netpod::Cluster;
@@ -24,11 +26,11 @@ async fn open_bytes_data_streams_http(
ctx: ReqCtx,
cluster: Cluster,
) -> Result<Vec<BoxedBytesStream>, Error> {
let frame1 = make_node_command_frame(subq.clone())?;
let frame1 = make_node_command_frame(subq.clone()).map_err(|e| Error::from_string(e))?;
let mut streams = Vec::new();
for node in &cluster.nodes {
let item = sitem_data(frame1.clone());
let buf = item.make_frame_dyn()?;
let buf = item.make_frame_dyn().map_err(|e| Error::from_string(e))?;
let url = node.baseurl().join("/api/4/private/eventdata/frames").unwrap();
debug!("open_event_data_streams_http post {url}");
@@ -84,8 +86,9 @@ impl OpenBoxedBytesStreams for OpenBoxedBytesViaHttp {
&self,
subq: EventsSubQuery,
ctx: ReqCtx,
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, Error>> + Send>> {
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, streams::tcprawclient::Error>> + Send>> {
let fut = open_bytes_data_streams_http(subq, ctx, self.cluster.clone());
let fut = fut.map_err(|e| streams::tcprawclient::Error::from(e.to_string()));
Box::pin(fut)
}
}

24
crates/nodenet/src/conn.rs
View File

@@ -4,7 +4,9 @@ use err::thiserror;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::LogItem;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
@@ -49,6 +51,11 @@ pub enum Error {
Items(#[from] items_2::Error),
NotAvailable,
DebugTest,
Generator(#[from] streams::generators::Error),
Transform(#[from] streams::transform::Error),
Framable(#[from] items_2::framable::Error),
Frame(#[from] items_2::frame::Error),
InMem(#[from] streams::frames::inmem::Error),
}
pub async fn events_service(ncc: NodeConfigCached) -> Result<(), Error> {
@@ -138,8 +145,7 @@ pub async fn create_response_bytes_stream(
// TODO support event blobs as transform
let fetch_info = evq.ch_conf().to_sf_databuffer()?;
let stream = disk::raw::conn::make_event_blobs_pipe(&evq, &fetch_info, reqctx, ncc)?;
// let stream = stream.map(|x| Box::new(x) as _);
let stream = stream.map(|x| x.make_frame_dyn().map(|x| x.freeze()));
let stream = stream.map(|x| x.make_frame_dyn().map(|x| x.freeze()).map_err(sitem_err2_from_string));
let ret = Box::pin(stream);
Ok(ret)
} else {
@@ -161,7 +167,11 @@ pub async fn create_response_bytes_stream(
})
});
// let stream = stream.map(move |x| Box::new(x) as Box<dyn Framable + Send>);
let stream = stream.map(|x| x.make_frame_dyn().map(bytes::BytesMut::freeze));
let stream = stream.map(|x| {
x.make_frame_dyn()
.map(bytes::BytesMut::freeze)
.map_err(sitem_err2_from_string)
});
let ret = Box::pin(stream);
Ok(ret)
}
@@ -324,13 +334,13 @@ async fn events_conn_handler_inner<INP>(
ncc: &NodeConfigCached,
) -> Result<(), Error>
where
INP: Stream<Item = Result<Bytes, err::Error>> + Unpin,
INP: Stream<Item = Result<Bytes, items_0::streamitem::SitemErrTy>> + Unpin,
{
match events_conn_handler_inner_try(netin, netout, addr, scyqueue, ncc).await {
Ok(_) => (),
Err(ce) => {
let mut out = ce.netout;
let item: Sitemty<ChannelEvents> = Err(err::Error::from_string(ce.err));
let item: Sitemty<ChannelEvents> = Err(items_0::streamitem::SitemErrTy::from_string(ce.err));
let buf = Framable::make_frame_dyn(&item)?;
out.write_all(&buf).await?;
}
@@ -345,7 +355,9 @@ async fn events_conn_handler(
ncc: NodeConfigCached,
) -> Result<(), Error> {
let (netin, netout) = stream.into_split();
let inp = Box::new(TcpReadAsBytes::new(netin));
let inp = TcpReadAsBytes::new(netin);
let inp = inp.map_err(sitem_err2_from_string);
let inp = Box::new(inp);
let span1 = span!(Level::INFO, "events_conn_handler");
let r = events_conn_handler_inner(inp, netout, addr, scyqueue, &ncc)
.instrument(span1)

1
crates/streams/src/cbor_stream.rs
View File

@@ -18,7 +18,6 @@ use netpod::log::Level;
use netpod::log::*;
use netpod::ScalarType;
use netpod::Shape;
use std::fmt;
use std::io::Cursor;
use std::pin::Pin;
use std::task::Context;

7
crates/streams/src/frames/eventsfromframes.rs
View File

@@ -3,6 +3,7 @@ use futures_util::StreamExt;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::sitem_err_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::SitemErrTy;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_2::frame::decode_frame;
@@ -19,7 +20,7 @@ use std::task::Poll;
pub enum Error {}
pub struct EventsFromFrames<O> {
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, SitemErrTy>> + Send>>,
dbgdesc: String,
errored: bool,
completed: bool,
@@ -28,7 +29,7 @@ pub struct EventsFromFrames<O> {
impl<O> EventsFromFrames<O> {
pub fn new(
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, SitemErrTy>> + Send>>,
dbgdesc: String,
) -> Self {
Self {
@@ -98,7 +99,7 @@ where
e
);
self.errored = true;
Ready(Some(Err(e)))
Ready(Some(sitem_err_from_string(e)))
}
},
},

21
crates/streams/src/generators.rs
View File

@@ -10,7 +10,6 @@ use items_0::on_sitemty_data;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::SitemErrTy;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
@@ -37,6 +36,8 @@ use std::time::Duration;
pub enum Error {
UnsupportedIsEventBlobs,
Transform(#[from] crate::transform::Error),
Items2(#[from] items_2::Error),
BadChannelName,
}
pub fn make_test_channel_events_bytes_stream(
@@ -68,7 +69,7 @@ pub fn make_test_channel_events_bytes_stream(
});
let stream = stream
.map_err(sitem_err2_from_string)
.map(|x| x.make_frame_dyn().map(|x| x.freeze()));
.map(|x| x.make_frame_dyn().map(|x| x.freeze()).map_err(sitem_err2_from_string));
let ret = Box::pin(stream);
Ok(ret)
}
@@ -106,14 +107,10 @@ fn make_test_channel_events_stream_data_inner(
} else {
let na: Vec<_> = chn.split("-").collect();
if na.len() != 3 {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
} else {
if na[0] != "inmem" {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
} else {
let _range = subq.range().clone();
if na[1] == "d0" {
@@ -124,14 +121,10 @@ fn make_test_channel_events_stream_data_inner(
//generator::generate_f32(node_ix, node_count, range)
panic!()
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
}
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
Err(Error::BadChannelName)
}
}
}

2
crates/streams/src/itemclone.rs
View File

@@ -65,7 +65,7 @@ where
pin_mut!(fut);
match fut.poll(cx) {
Ready(Ok(())) => Ready(Ok(())),
Ready(Err(e)) => Ready(Err(e.into())),
Ready(Err(_)) => todo!("can not send copy"),
Pending => Pending,
}
}

25
crates/streams/src/json_stream.rs
View File

@@ -2,6 +2,7 @@ use crate::cbor_stream::SitemtyDynEventsStream;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Events;
use items_0::WithLen;
@@ -11,7 +12,23 @@ use std::time::Duration;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "JsonStream")]
pub enum Error {}
pub enum Error {
Json(#[from] serde_json::Error),
Msg(String),
}
pub struct ErrMsg<E>(pub E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
pub struct JsonBytes(String);
@@ -56,7 +73,7 @@ pub fn events_stream_to_json_stream(stream: SitemtyDynEventsStream) -> impl Stre
prepend.chain(stream)
}
fn map_events(x: Result<StreamItem<RangeCompletableItem<Box<dyn Events>>>, Error>) -> Result<JsonBytes, Error> {
fn map_events(x: Sitemty<Box<dyn Events>>) -> Result<JsonBytes, Error> {
match x {
Ok(x) => match x {
StreamItem::DataItem(x) => match x {
@@ -104,12 +121,12 @@ fn map_events(x: Result<StreamItem<RangeCompletableItem<Box<dyn Events>>>, Error
}
},
StreamItem::Log(item) => {
info!("{item:?}");
debug!("{item:?}");
let item = JsonBytes::new(String::new());
Ok(item)
}
StreamItem::Stats(item) => {
info!("{item:?}");
debug!("{item:?}");
let item = JsonBytes::new(String::new());
Ok(item)
}

10
crates/streams/src/needminbuffer.rs
View File

@@ -12,7 +12,7 @@ use std::task::Poll;
pub enum Error {}
pub struct NeedMinBuffer {
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, Error>> + Send>>,
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>> + Send>>,
need_min: u32,
left: Option<FileChunkRead>,
buf_len_histo: HistoLog2,
@@ -21,9 +21,11 @@ pub struct NeedMinBuffer {
}
impl NeedMinBuffer {
pub fn new(inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, Error>> + Send>>) -> Self {
pub fn new(
inp: Pin<Box<dyn Stream<Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>> + Send>>,
) -> Self {
Self {
inp: inp,
inp,
need_min: 1,
left: None,
buf_len_histo: HistoLog2::new(8),
@@ -50,7 +52,7 @@ impl Drop for NeedMinBuffer {
}
impl Stream for NeedMinBuffer {
type Item = Result<FileChunkRead, Error>;
type Item = Result<FileChunkRead, items_0::streamitem::SitemErrTy>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;

1
crates/streams/src/plaineventsjson.rs
View File

@@ -24,6 +24,7 @@ use std::time::Instant;
pub enum Error {
Stream(#[from] crate::plaineventsstream::Error),
Json(#[from] serde_json::Error),
Collect(#[from] crate::collect::Error),
}
pub async fn plain_events_json(

3
crates/streams/src/plaineventsstream.rs
View File

@@ -20,6 +20,7 @@ use std::pin::Pin;
pub enum Error {
Netpod(#[from] netpod::NetpodError),
Transform(#[from] crate::transform::Error),
TcpRawClient(#[from] crate::tcprawclient::Error),
}
pub type DynEventsStream = Pin<Box<dyn Stream<Item = Sitemty<Box<dyn Events>>> + Send>>;
@@ -77,7 +78,7 @@ pub async fn dyn_events_stream(
});
if let Some(wasmname) = evq.test_do_wasm() {
let stream = transform_wasm(stream, wasmname, ctx).await?;
let stream = transform_wasm::<_, items_0::streamitem::SitemErrTy>(stream, wasmname, ctx).await?;
Ok(Box::pin(stream))
} else {
Ok(Box::pin(stream))

28
crates/streams/src/rangefilter2.rs
View File

@@ -1,6 +1,6 @@
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::sitem_err_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StatsItem;
@@ -22,13 +22,12 @@ use items_0::Events;
#[cfg(test)]
use std::collections::VecDeque;
#[allow(unused)]
macro_rules! trace_emit {
($det:expr, $($arg:tt)*) => {
if $det {
eprintln!($($arg)*);
}
};
macro_rules! trace_emit { ($det:expr, $($arg:tt)*) => ( if false && $det { trace!($($arg)*); } ) }
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Rangefilter")]
pub enum Error {
Merge(#[from] MergeError),
}
pub struct RangeFilter2<S, ITY>
@@ -146,8 +145,7 @@ where
} else {
trace_emit!(self.trdet, "discarding events len {:?}", lige - 1);
let mut dummy = item.new_empty();
item.drain_into(&mut dummy, (0, lige - 1))
.map_err(|e| format!("{e} unexpected MergeError while remove of items"))?;
item.drain_into(&mut dummy, (0, lige - 1))?;
self.slot1 = None;
item
}
@@ -157,8 +155,7 @@ where
trace_emit!(self.trdet, "drain into to keep one before");
let n = item.len();
let mut keep = item.new_empty();
item.drain_into(&mut keep, (n.max(1) - 1, n))
.map_err(|e| format!("{e} unexpected MergeError while remove of items"))?;
item.drain_into(&mut keep, (n.max(1) - 1, n))?;
self.slot1 = Some(keep);
item.new_empty()
}
@@ -192,10 +189,7 @@ where
loop {
break if self.complete {
error!("{} poll_next on complete", Self::type_name());
Ready(Some(Err(Error::with_msg_no_trace(format!(
"{} poll_next on complete",
Self::type_name()
)))))
Ready(Some(sitem_err_from_string("poll next on complete")))
} else if self.done {
self.complete = true;
Ready(None)
@@ -223,7 +217,7 @@ where
Err(e) => {
error!("sees: {e}");
self.inp_done = true;
Ready(Some(Err(e)))
Ready(Some(sitem_err_from_string(e)))
}
},
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)) => {

66
crates/streams/src/tcprawclient.rs
View File

@@ -7,14 +7,15 @@ use crate::frames::eventsfromframes::EventsFromFrames;
use crate::frames::inmem::BoxedBytesStream;
use crate::frames::inmem::InMemoryFrameStream;
use crate::frames::inmem::TcpReadAsBytes;
use err::Error;
use futures_util::Future;
use futures_util::Stream;
use futures_util::TryStreamExt;
use http::Uri;
use httpclient::body_bytes;
use httpclient::http;
use items_0::framable::FrameTypeInnerStatic;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::Sitemty;
use items_2::eventfull::EventFull;
use items_2::framable::EventQueryJsonStringFrame;
@@ -42,6 +43,40 @@ use tokio::net::TcpStream;
pub const TEST_BACKEND: &str = "testbackend-00";
#[derive(Debug, thiserror::Error)]
#[cstm(name = "TcpRawClient")]
pub enum Error {
IO(#[from] std::io::Error),
Msg(String),
Frame(#[from] items_2::frame::Error),
Framable(#[from] items_2::framable::Error),
Json(#[from] serde_json::Error),
Http(#[from] http::Error),
HttpClient(#[from] httpclient::Error),
Hyper(#[from] httpclient::hyper::Error),
#[error("ServerError({0:?}, {1})")]
ServerError(http::response::Parts, String),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
impl From<String> for Error {
fn from(value: String) -> Self {
Self::Msg(value)
}
}
pub trait OpenBoxedBytesStreams {
fn open(
&self,
@@ -75,8 +110,10 @@ pub async fn x_processed_event_blobs_stream_from_node_tcp(
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
let inp = Box::pin(TcpReadAsBytes::new(netin)) as BoxedBytesStream;
let inp = TcpReadAsBytes::new(netin).map_err(sitem_err2_from_string);
let inp = Box::pin(inp) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let items = EventsFromFrames::new(frames, addr);
Ok(Box::pin(items))
@@ -106,31 +143,20 @@ pub async fn x_processed_event_blobs_stream_from_node_http(
.header(header::HOST, uri.host().unwrap())
.header(header::ACCEPT, APP_OCTET)
.header(ctx.header_name(), ctx.header_value())
.body(body_bytes(buf))
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
.body(body_bytes(buf))?;
let mut client = httpclient::connect_client(req.uri()).await?;
let res = client
.send_request(req)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let res = client.send_request(req).await?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = httpclient::read_body_bytes(body).await?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
return Err(Error::ServerError(head, s.to_string()));
}
let (_head, body) = res.into_parts();
let inp = Box::pin(httpclient::IncomingStream::new(body)) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let stream = EventsFromFrames::new(frames, url.to_string());
debug!("open_event_data_streams_http done {url}");
@@ -175,8 +201,11 @@ where
netout.flush().await?;
netout.forget();
// TODO for images, we need larger buffer capacity
let inp = Box::pin(TcpReadAsBytes::new(netin)) as BoxedBytesStream;
let inp = TcpReadAsBytes::new(netin);
let inp = inp.map_err(sitem_err2_from_string);
let inp = Box::pin(inp) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
streams.push(Box::pin(stream) as _);
@@ -193,6 +222,7 @@ where
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
let frames = InMemoryFrameStream::new(inp, bufcap);
let frames = frames.map_err(sitem_err2_from_string);
// TODO let EventsFromFrames accept also non-boxed input?
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, dbgdesc);

12
crates/streams/src/test.rs
View File

@@ -5,7 +5,6 @@ mod events;
#[cfg(test)]
mod timebin;
use err::Error;
use futures_util::stream;
use futures_util::Stream;
use items_0::streamitem::sitem_data;
@@ -17,6 +16,10 @@ use items_2::eventsdim0::EventsDim0;
use netpod::timeunits::SEC;
use std::pin::Pin;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "StreamsTest")]
pub enum Error {}
type BoxedEventStream = Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>;
// TODO use some xorshift generator.
@@ -51,11 +54,10 @@ fn merge_mergeable_00() -> Result<(), Error> {
runfut(fut)
}
fn runfut<T, F>(fut: F) -> Result<T, err::Error>
fn runfut<F, T, E>(fut: F) -> Result<T, E>
where
F: std::future::Future<Output = Result<T, Error>>,
F: std::future::Future<Output = Result<T, E>>,
E: std::error::Error,
{
use futures_util::TryFutureExt;
let fut = fut.map_err(|e| e.into());
taskrun::run(fut)
}

5
crates/streams/src/test/events.rs
View File

@@ -8,6 +8,7 @@ use crate::tcprawclient::TEST_BACKEND;
use futures_util::future;
use futures_util::Future;
use futures_util::StreamExt;
use futures_util::TryFutureExt;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange;
@@ -83,8 +84,8 @@ impl OpenBoxedBytesStreams for StreamOpener {
&self,
subq: EventsSubQuery,
_ctx: ReqCtx,
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, Error>> + Send>> {
Box::pin(stream_opener(subq))
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, crate::tcprawclient::Error>> + Send>> {
Box::pin(stream_opener(subq).map_err(|e| crate::tcprawclient::Error::Msg(format!("{e}"))))
}
}

7
crates/streams/src/teststream.rs
View File

@@ -1,21 +1,20 @@
use crate::timebin::cached::reader::EventsReadProvider;
use crate::timebin::cached::reader::EventsReading;
use futures_util::Stream;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::Sitemty;
use items_2::channelevents::ChannelEvents;
use netpod::range::evrange::SeriesRange;
use query::api4::events::EventsSubQuery;
use rand_xoshiro::rand_core::SeedableRng;
use rand_xoshiro::Xoshiro128PlusPlus;
use std::pin::Pin;
fn make_stream(chname: &str, range: &SeriesRange) -> Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>> {
if chname == "unittest;scylla;cont;scalar;f32" {
let e = ::err::Error::with_msg_no_trace("unknown channel {chname}");
let e = sitem_err2_from_string(format!("unknown channel {chname}"));
let ret = futures_util::stream::iter([Err(e)]);
Box::pin(ret)
} else {
let e = ::err::Error::with_msg_no_trace("unknown channel {chname}");
let e = sitem_err2_from_string(format!("unknown channel {chname}"));
let ret = futures_util::stream::iter([Err(e)]);
Box::pin(ret)
}

20
crates/streams/src/timebinnedjson.rs
View File

@@ -14,6 +14,7 @@ use futures_util::Stream;
use futures_util::StreamExt;
use items_0::collect_s::CollectableDyn;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
@@ -41,6 +42,23 @@ pub enum Error {
Query(#[from] query::api4::binned::Error),
FromLayers(#[from] super::timebin::fromlayers::Error),
Transform(#[from] super::transform::Error),
TcpRawClient(#[from] crate::tcprawclient::Error),
Collect(#[from] crate::collect::Error),
Json(#[from] serde_json::Error),
Msg(String),
}
struct ErrMsg<E>(E)
where
E: ToString;
impl<E> From<ErrMsg<E>> for Error
where
E: ToString,
{
fn from(value: ErrMsg<E>) -> Self {
Self::Msg(value.0.to_string())
}
}
#[allow(unused)]
@@ -435,7 +453,7 @@ pub async fn timebinned_json_framed(
// TODO skip the intermediate conversion to js value, go directly to string data
let stream = stream.map(|x| match x {
Ok(x) => Ok(JsonBytes::new(serde_json::to_string(&x).unwrap())),
Err(e) => Err(e),
Err(e) => Err(crate::json_stream::Error::from(crate::json_stream::ErrMsg(e))),
});
Ok(Box::pin(stream))
}

17
crates/streams/src/transform.rs
View File

@@ -20,23 +20,20 @@ use std::pin::Pin;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Transform")]
pub enum Error {}
pub enum Error {
#[error("UnhandledQuery({0:?})")]
UnhandledQuery(EventTransformQuery),
}
pub fn build_event_transform(tr: &TransformQuery) -> Result<TransformEvent, Error> {
let trev = tr.get_tr_event();
match trev {
EventTransformQuery::ValueFull => Ok(make_transform_identity()),
EventTransformQuery::MinMaxAvgDev => Ok(make_transform_min_max_avg()),
EventTransformQuery::ArrayPick(..) => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::ArrayPick(..) => Err(Error::UnhandledQuery(trev.clone())),
EventTransformQuery::PulseIdDiff => Ok(make_transform_pulse_id_diff()),
EventTransformQuery::EventBlobsVerbatim => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::EventBlobsUncompressed => Err(Error::with_msg_no_trace(format!(
"build_event_transform don't know what to do {trev:?}"
))),
EventTransformQuery::EventBlobsVerbatim => Err(Error::UnhandledQuery(trev.clone())),
EventTransformQuery::EventBlobsUncompressed => Err(Error::UnhandledQuery(trev.clone())),
}
}