daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on sf databuffer channel config decision

Browse Source
This commit is contained in:
Dominik Werder
2023-06-23 16:31:55 +02:00
parent 0260f4b4d6
commit 64d2996c48
26 changed files with 249 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
streams/src/frames/inmem.rs
View File

@@ -10,6 +10,7 @@ use items_2::framable::INMEM_FRAME_HEAD;
use items_2::framable::INMEM_FRAME_MAGIC;
use items_2::inmem::InMemoryFrame;
use netpod::log::*;
use netpod::ByteSize;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -51,10 +52,10 @@ where
std::any::type_name::<Self>()
}
pub fn new(inp: T, bufcap: usize) -> Self {
pub fn new(inp: T, bufcap: ByteSize) -> Self {
Self {
inp,
buf: SlideBuf::new(bufcap),
buf: SlideBuf::new(bufcap.bytes() as usize),
need_min: INMEM_FRAME_HEAD,
done: false,
complete: false,

16
streams/src/tcprawclient.rs
View File

@@ -20,7 +20,6 @@ use netpod::log::*;
use netpod::ChannelTypeConfigGen;
use netpod::Cluster;
use netpod::Node;
use netpod::PerfOpts;
use query::api4::events::EventsSubQuery;
use query::api4::events::Frame1Parts;
use serde::de::DeserializeOwned;
@@ -36,14 +35,12 @@ pub fn make_node_command_frame(query: EventsSubQuery) -> Result<EventQueryJsonSt
}
pub async fn x_processed_event_blobs_stream_from_node(
query: EventsSubQuery,
ch_conf: ChannelTypeConfigGen,
perf_opts: PerfOpts,
subq: EventsSubQuery,
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
let addr = format!("{}:{}", node.host, node.port_raw);
debug!("x_processed_event_blobs_stream_from_node to: {addr}",);
let frame1 = make_node_command_frame(query)?;
let frame1 = make_node_command_frame(subq.clone())?;
let net = TcpStream::connect(addr.clone()).await?;
let (netin, mut netout) = net.into_split();
let item = sitem_data(frame1);
@@ -53,7 +50,7 @@ pub async fn x_processed_event_blobs_stream_from_node(
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let frames = InMemoryFrameAsyncReadStream::new(netin, subq.inmem_bufcap());
let frames = Box::pin(frames);
let items = EventsFromFrames::new(frames, addr);
Ok(Box::pin(items))
@@ -61,13 +58,13 @@ pub async fn x_processed_event_blobs_stream_from_node(
pub type BoxedStream<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
pub async fn open_tcp_streams<T>(query: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
pub async fn open_tcp_streams<T>(subq: EventsSubQuery, cluster: &Cluster) -> Result<Vec<BoxedStream<T>>, Error>
where
// Group bounds in new trait
T: FrameTypeInnerStatic + DeserializeOwned + Send + Unpin + fmt::Debug + 'static,
{
// TODO when unit tests established, change to async connect:
let frame1 = make_node_command_frame(query)?;
let frame1 = make_node_command_frame(subq.clone())?;
let mut streams = Vec::new();
for node in &cluster.nodes {
let addr = format!("{}:{}", node.host, node.port_raw);
@@ -82,8 +79,7 @@ where
netout.flush().await?;
netout.forget();
// TODO for images, we need larger buffer capacity
let perf_opts = PerfOpts::default();
let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let frames = InMemoryFrameAsyncReadStream::new(netin, subq.inmem_bufcap());
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
streams.push(Box::pin(stream) as _);