daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Read internal event data frames also through http

Browse Source
This commit is contained in:
Dominik Werder
2023-08-17 14:36:04 +02:00
parent 92e58291f3
commit 954cff05f0
19 changed files with 460 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/streams/src/frames/inmem.rs
+70 -16
View File
@@ -15,7 +15,6 @@ use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use tokio::io::AsyncRead;
use tokio::io::ReadBuf;
#[allow(unused)]
macro_rules! trace2 {
@@ -29,14 +28,54 @@ impl err::ToErr for crate::slidebuf::Error {
}
}
pub struct TcpReadAsBytes<INP> {
inp: INP,
}
impl<INP> TcpReadAsBytes<INP> {
pub fn new(inp: INP) -> Self {
Self { inp }
}
}
impl<INP> Stream for TcpReadAsBytes<INP>
where
INP: AsyncRead + Unpin,
{
type Item = Result<Bytes, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
// TODO keep this small as long as InMemoryFrameStream uses SlideBuf internally.
let mut buf1 = vec![0; 128];
let mut buf2 = tokio::io::ReadBuf::new(&mut buf1);
match tokio::io::AsyncRead::poll_read(Pin::new(&mut self.inp), cx, &mut buf2) {
Ready(Ok(())) => {
let n = buf2.filled().len();
if n == 0 {
Ready(None)
} else {
buf1.truncate(n);
let item = Bytes::from(buf1);
Ready(Some(Ok(item)))
}
}
Ready(Err(e)) => Ready(Some(Err(e.into()))),
Pending => Pending,
}
}
}
/// Interprets a byte stream as length-delimited frames.
///
/// Emits each frame as a single item. Therefore, each item must fit easily into memory.
pub struct InMemoryFrameAsyncReadStream<T>
pub struct InMemoryFrameStream<T, E>
where
T: AsyncRead + Unpin,
T: Stream<Item = Result<Bytes, E>> + Unpin,
{
inp: T,
// TODO since we moved to input stream of Bytes, we have the danger that the ring buffer
// is not large enough. Actually, this should rather use a RopeBuf with incoming owned bufs.
buf: SlideBuf,
need_min: usize,
done: bool,
@@ -44,9 +83,9 @@ where
inp_bytes_consumed: u64,
}
impl<T> InMemoryFrameAsyncReadStream<T>
impl<T, E> InMemoryFrameStream<T, E>
where
T: AsyncRead + Unpin,
T: Stream<Item = Result<Bytes, E>> + Unpin,
{
pub fn type_name() -> &'static str {
std::any::type_name::<Self>()
@@ -66,20 +105,35 @@ where
fn poll_upstream(&mut self, cx: &mut Context) -> Poll<Result<usize, Error>> {
trace2!("poll_upstream");
use Poll::*;
let mut buf = ReadBuf::new(self.buf.available_writable_area(self.need_min - self.buf.len())?);
// use tokio::io::AsyncRead;
// use tokio::io::ReadBuf;
// let mut buf = ReadBuf::new(self.buf.available_writable_area(self.need_min.saturating_sub(self.buf.len()))?);
let inp = &mut self.inp;
pin_mut!(inp);
trace!("poll_upstream");
match AsyncRead::poll_read(inp, cx, &mut buf) {
Ready(Ok(())) => {
let n = buf.filled().len();
self.buf.wadv(n)?;
trace2!("recv bytes {}", n);
Ready(Ok(n))
}
Ready(Err(e)) => Ready(Err(e.into())),
match inp.poll_next(cx) {
Ready(Some(Ok(x))) => match self.buf.available_writable_area(x.len()) {
Ok(dst) => {
dst[..x.len()].copy_from_slice(&x);
self.buf.wadv(x.len())?;
Ready(Ok(x.len()))
}
Err(e) => Ready(Err(e.into())),
},
Ready(Some(Err(_e))) => Ready(Err(Error::with_msg_no_trace("input error"))),
Ready(None) => Ready(Ok(0)),
Pending => Pending,
}
// match AsyncRead::poll_read(inp, cx, &mut buf) {
// Ready(Ok(())) => {
// let n = buf.filled().len();
// self.buf.wadv(n)?;
// trace2!("recv bytes {}", n);
// Ready(Ok(n))
// }
// Ready(Err(e)) => Ready(Err(e.into())),
// Pending => Pending,
// }
}
// Try to consume bytes to parse a frame.
@@ -157,9 +211,9 @@ where
}
}
impl<T> Stream for InMemoryFrameAsyncReadStream<T>
impl<T, E> Stream for InMemoryFrameStream<T, E>
where
T: AsyncRead + Unpin,
T: Stream<Item = Result<Bytes, E>> + Unpin,
{
type Item = Result<StreamItem<InMemoryFrame>, Error>;
crates/streams/src/plaineventsjson.rs
+2 -2
View File
@@ -1,5 +1,5 @@
use crate::collect::Collect;
use crate::tcprawclient::open_tcp_streams;
use crate::tcprawclient::open_event_data_streams;
use crate::transform::build_merged_event_transform;
use crate::transform::EventsToTimeBinnable;
use crate::transform::TimeBinnableToCollectable;
@@ -35,7 +35,7 @@ pub async fn plain_events_json(
let deadline = Instant::now() + evq.timeout();
let mut tr = build_merged_event_transform(evq.transform())?;
// TODO make sure the empty container arrives over the network.
let inps = open_tcp_streams::<ChannelEvents>(subq, cluster).await?;
let inps = open_event_data_streams::<ChannelEvents>(subq, cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader.
// TODO use a mixture of count and byte-size as threshold.
let stream = Merger::new(inps, evq.merger_out_len_max());
crates/streams/src/tcprawclient.rs
+147 -12
View File
@@ -1,12 +1,11 @@
/*!
Delivers event data.
Delivers event data (not yet time-binned) from local storage and provides client functions
to request such data from nodes.
*/
//! Delivers event data.
//!
//! Delivers event data (not yet time-binned) from local storage and provides client functions
//! to request such data from nodes.
use crate::frames::eventsfromframes::EventsFromFrames;
use crate::frames::inmem::InMemoryFrameAsyncReadStream;
use crate::frames::inmem::InMemoryFrameStream;
use crate::frames::inmem::TcpReadAsBytes;
use err::Error;
use futures_util::Stream;
use items_0::framable::FrameTypeInnerStatic;
@@ -33,7 +32,7 @@ pub fn make_node_command_frame(query: EventsSubQuery) -> Result<EventQueryJsonSt
Ok(EventQueryJsonStringFrame(ret))
}
pub async fn x_processed_event_blobs_stream_from_node(
pub async fn x_processed_event_blobs_stream_from_node_tcp(
subq: EventsSubQuery,
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
@@ -49,17 +48,82 @@ pub async fn x_processed_event_blobs_stream_from_node(
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
let frames = InMemoryFrameAsyncReadStream::new(netin, subq.inmem_bufcap());
let frames = InMemoryFrameStream::new(TcpReadAsBytes::new(netin), subq.inmem_bufcap());
let frames = Box::pin(frames);
let items = EventsFromFrames::new(frames, addr);
Ok(Box::pin(items))
}
pub async fn x_processed_event_blobs_stream_from_node_http(
subq: EventsSubQuery,
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
use http::header;
use http::Method;
use http::Request;
use httpclient::http;
use httpclient::hyper;
use hyper::Body;
use hyper::StatusCode;
let frame1 = make_node_command_frame(subq.clone())?;
let item = sitem_data(frame1.clone());
let buf = item.make_frame()?;
let url = node.baseurl().join("/api/4/private/eventdata/frames").unwrap();
debug!("open_event_data_streams_http post {url}");
let req = Request::builder()
.method(Method::POST)
.uri(url.to_string())
.header(header::ACCEPT, "application/octet-stream")
.body(Body::from(buf.to_vec()))
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let client = hyper::Client::new();
let res = client
.request(req)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = hyper::body::to_bytes(body)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
}
let (_head, body) = res.into_parts();
let frames = InMemoryFrameStream::new(body, subq.inmem_bufcap());
let frames = Box::pin(frames);
let stream = EventsFromFrames::new(frames, url.to_string());
debug!("open_event_data_streams_http done {url}");
Ok(Box::pin(stream))
}
pub async fn x_processed_event_blobs_stream_from_node(
subq: EventsSubQuery,
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
if true {
x_processed_event_blobs_stream_from_node_http(subq, node).await
} else {
x_processed_event_blobs_stream_from_node_tcp(subq, node).await
}
}
pub type BoxedStream<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
pub async fn open_tcp_streams<T>(subq: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
async fn open_event_data_streams_tcp<T>(subq: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
where
// Group bounds in new trait
// TODO group bounds in new trait
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
// TODO when unit tests established, change to async connect:
@@ -78,10 +142,81 @@ where
netout.flush().await?;
netout.forget();
// TODO for images, we need larger buffer capacity
let frames = InMemoryFrameAsyncReadStream::new(netin, subq.inmem_bufcap());
let frames = InMemoryFrameStream::new(TcpReadAsBytes::new(netin), subq.inmem_bufcap());
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
streams.push(Box::pin(stream) as _);
}
Ok(streams)
}
async fn open_event_data_streams_http<T>(subq: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
where
// TODO group bounds in new trait
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
let frame1 = make_node_command_frame(subq.clone())?;
let mut streams = Vec::new();
for node in &cluster.nodes {
use http::header;
use http::Method;
use http::Request;
use httpclient::http;
use httpclient::hyper;
use hyper::Body;
use hyper::StatusCode;
let item = sitem_data(frame1.clone());
let buf = item.make_frame()?;
let url = node.baseurl().join("/api/4/private/eventdata/frames").unwrap();
debug!("open_event_data_streams_http post {url}");
let req = Request::builder()
.method(Method::POST)
.uri(url.to_string())
.header(header::ACCEPT, "application/octet-stream")
.body(Body::from(buf.to_vec()))
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let client = hyper::Client::new();
let res = client
.request(req)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = hyper::body::to_bytes(body)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
}
let (_head, body) = res.into_parts();
let frames = InMemoryFrameStream::new(body, subq.inmem_bufcap());
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, url.to_string());
debug!("open_event_data_streams_http done {url}");
streams.push(Box::pin(stream) as _);
}
Ok(streams)
}
pub async fn open_event_data_streams<T>(subq: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
where
// TODO group bounds in new trait
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
if true {
open_event_data_streams_http(subq, cluster).await
} else {
open_event_data_streams_tcp(subq, cluster).await
}
}
crates/streams/src/timebinnedjson.rs
+2 -2
View File
@@ -1,6 +1,6 @@
use crate::collect::Collect;
use crate::rangefilter2::RangeFilter2;
use crate::tcprawclient::open_tcp_streams;
use crate::tcprawclient::open_event_data_streams;
use crate::timebin::TimeBinnedStream;
use crate::transform::build_merged_event_transform;
use crate::transform::EventsToTimeBinnable;
@@ -48,7 +48,7 @@ async fn timebinnable_stream(
let settings = EventsSubQuerySettings::from(&query);
let subq = EventsSubQuery::from_parts(select, settings, reqid);
let mut tr = build_merged_event_transform(subq.transform())?;
let inps = open_tcp_streams::<ChannelEvents>(subq, &cluster).await?;
let inps = open_event_data_streams::<ChannelEvents>(subq, &cluster).await?;
// TODO propagate also the max-buf-len for the first stage event reader.
// TODO use a mixture of count and byte-size as threshold.
let stream = Merger::new(inps, query.merger_out_len_max());