daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-11-05 14:27:41 +01:00
parent 35d15c4694
commit ef021ff971
31 changed files with 266 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/nodenet/src/client.rs
View File

@@ -1,5 +1,6 @@
use err::Error;
use futures_util::Future;
use futures_util::TryFutureExt;
use http::header;
use http::Method;
use http::Request;
@@ -8,6 +9,7 @@ use httpclient::http;
use httpclient::hyper::StatusCode;
use httpclient::hyper::Uri;
use items_0::streamitem::sitem_data;
use items_0::streamitem::sitem_err2_from_string;
use items_2::framable::Framable;
use netpod::log::*;
use netpod::Cluster;
@@ -24,11 +26,11 @@ async fn open_bytes_data_streams_http(
ctx: ReqCtx,
cluster: Cluster,
) -> Result<Vec<BoxedBytesStream>, Error> {
let frame1 = make_node_command_frame(subq.clone())?;
let frame1 = make_node_command_frame(subq.clone()).map_err(|e| Error::from_string(e))?;
let mut streams = Vec::new();
for node in &cluster.nodes {
let item = sitem_data(frame1.clone());
let buf = item.make_frame_dyn()?;
let buf = item.make_frame_dyn().map_err(|e| Error::from_string(e))?;
let url = node.baseurl().join("/api/4/private/eventdata/frames").unwrap();
debug!("open_event_data_streams_http post {url}");
@@ -84,8 +86,9 @@ impl OpenBoxedBytesStreams for OpenBoxedBytesViaHttp {
&self,
subq: EventsSubQuery,
ctx: ReqCtx,
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, Error>> + Send>> {
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, streams::tcprawclient::Error>> + Send>> {
let fut = open_bytes_data_streams_http(subq, ctx, self.cluster.clone());
let fut = fut.map_err(|e| streams::tcprawclient::Error::from(e.to_string()));
Box::pin(fut)
}
}

24
crates/nodenet/src/conn.rs
View File

@@ -4,7 +4,9 @@ use err::thiserror;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use items_0::on_sitemty_data;
use items_0::streamitem::sitem_err2_from_string;
use items_0::streamitem::LogItem;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
@@ -49,6 +51,11 @@ pub enum Error {
Items(#[from] items_2::Error),
NotAvailable,
DebugTest,
Generator(#[from] streams::generators::Error),
Transform(#[from] streams::transform::Error),
Framable(#[from] items_2::framable::Error),
Frame(#[from] items_2::frame::Error),
InMem(#[from] streams::frames::inmem::Error),
}
pub async fn events_service(ncc: NodeConfigCached) -> Result<(), Error> {
@@ -138,8 +145,7 @@ pub async fn create_response_bytes_stream(
// TODO support event blobs as transform
let fetch_info = evq.ch_conf().to_sf_databuffer()?;
let stream = disk::raw::conn::make_event_blobs_pipe(&evq, &fetch_info, reqctx, ncc)?;
// let stream = stream.map(|x| Box::new(x) as _);
let stream = stream.map(|x| x.make_frame_dyn().map(|x| x.freeze()));
let stream = stream.map(|x| x.make_frame_dyn().map(|x| x.freeze()).map_err(sitem_err2_from_string));
let ret = Box::pin(stream);
Ok(ret)
} else {
@@ -161,7 +167,11 @@ pub async fn create_response_bytes_stream(
})
});
// let stream = stream.map(move |x| Box::new(x) as Box<dyn Framable + Send>);
let stream = stream.map(|x| x.make_frame_dyn().map(bytes::BytesMut::freeze));
let stream = stream.map(|x| {
x.make_frame_dyn()
.map(bytes::BytesMut::freeze)
.map_err(sitem_err2_from_string)
});
let ret = Box::pin(stream);
Ok(ret)
}
@@ -324,13 +334,13 @@ async fn events_conn_handler_inner<INP>(
ncc: &NodeConfigCached,
) -> Result<(), Error>
where
INP: Stream<Item = Result<Bytes, err::Error>> + Unpin,
INP: Stream<Item = Result<Bytes, items_0::streamitem::SitemErrTy>> + Unpin,
{
match events_conn_handler_inner_try(netin, netout, addr, scyqueue, ncc).await {
Ok(_) => (),
Err(ce) => {
let mut out = ce.netout;
let item: Sitemty<ChannelEvents> = Err(err::Error::from_string(ce.err));
let item: Sitemty<ChannelEvents> = Err(items_0::streamitem::SitemErrTy::from_string(ce.err));
let buf = Framable::make_frame_dyn(&item)?;
out.write_all(&buf).await?;
}
@@ -345,7 +355,9 @@ async fn events_conn_handler(
ncc: NodeConfigCached,
) -> Result<(), Error> {
let (netin, netout) = stream.into_split();
let inp = Box::new(TcpReadAsBytes::new(netin));
let inp = TcpReadAsBytes::new(netin);
let inp = inp.map_err(sitem_err2_from_string);
let inp = Box::new(inp);
let span1 = span!(Level::INFO, "events_conn_handler");
let r = events_conn_handler_inner(inp, netout, addr, scyqueue, &ncc)
.instrument(span1)