daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2023-02-24 13:32:19 +01:00
parent 0f29eac2b5
commit 2e286eb28e
23 changed files with 399 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
streams/src/eventchunker.rs
View File

@@ -40,7 +40,6 @@ pub struct EventChunker {
seen_after_range_count: usize,
unordered_warn_count: usize,
repeated_ts_warn_count: usize,
dbgdesc: String,
}
impl Drop for EventChunker {
@@ -84,7 +83,6 @@ impl EventChunker {
dbg_path: PathBuf,
expand: bool,
do_decompress: bool,
dbgdesc: String,
) -> Self {
trace!("EventChunker::from_start");
let mut inp = NeedMinBuffer::new(inp);
@@ -113,7 +111,6 @@ impl EventChunker {
seen_after_range_count: 0,
unordered_warn_count: 0,
repeated_ts_warn_count: 0,
dbgdesc,
}
}
@@ -126,18 +123,8 @@ impl EventChunker {
dbg_path: PathBuf,
expand: bool,
do_decompress: bool,
dbgdesc: String,
) -> Self {
let mut ret = Self::from_start(
inp,
channel_config,
range,
stats_conf,
dbg_path,
expand,
do_decompress,
dbgdesc,
);
let mut ret = Self::from_start(inp, channel_config, range, stats_conf, dbg_path, expand, do_decompress);
ret.state = DataFileState::Event;
ret.need_min = 4;
ret.inp.set_need_min(4);

31
streams/src/frames/eventsfromframes.rs
View File

@@ -1,7 +1,8 @@
use super::inmem::InMemoryFrameAsyncReadStream;
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
use items::frame::decode_frame;
use items::inmem::InMemoryFrame;
use items::FrameTypeInnerStatic;
use items::Sitemty;
use items::StreamItem;
@@ -11,23 +12,16 @@ use std::marker::PhantomData;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use tokio::io::AsyncRead;
pub struct EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
{
inp: InMemoryFrameAsyncReadStream<T>,
pub struct EventsFromFrames<O> {
inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>,
errored: bool,
completed: bool,
_m1: PhantomData<I>,
_m1: PhantomData<O>,
}
impl<T, I> EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
{
pub fn new(inp: InMemoryFrameAsyncReadStream<T>) -> Self {
impl<O> EventsFromFrames<O> {
pub fn new(inp: Pin<Box<dyn Stream<Item = Result<StreamItem<InMemoryFrame>, Error>> + Send>>) -> Self {
Self {
inp,
errored: false,
@@ -37,15 +31,16 @@ where
}
}
impl<T, I> Stream for EventsFromFrames<T, I>
impl<O> Stream for EventsFromFrames<O>
where
T: AsyncRead + Unpin,
I: FrameTypeInnerStatic + DeserializeOwned + Unpin,
O: FrameTypeInnerStatic + DeserializeOwned + Unpin,
{
type Item = Sitemty<I>;
type Item = Sitemty<O>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let span = netpod::log::span!(netpod::log::Level::INFO, "EvFrFr");
let _spg = span.enter();
loop {
break if self.completed {
panic!("poll_next on completed");
@@ -57,7 +52,7 @@ where
Ready(Some(Ok(item))) => match item {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(frame) => match decode_frame::<Sitemty<I>>(&frame) {
StreamItem::DataItem(frame) => match decode_frame::<Sitemty<O>>(&frame) {
Ok(item) => match item {
Ok(item) => Ready(Some(Ok(item))),
Err(e) => {

3
streams/src/frames/inmem.rs
View File

@@ -9,6 +9,7 @@ use netpod::log::*;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
use tracing::Instrument;
#[allow(unused)]
macro_rules! trace2 {
@@ -152,7 +153,7 @@ where
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let span = span!(Level::TRACE, "inmem");
let span = span!(Level::INFO, "InMemRd");
let _spanguard = span.enter();
loop {
break if self.complete {

30
streams/src/slidebuf.rs
View File

@@ -93,6 +93,9 @@ impl SlideBuf {
pub fn wadv(&mut self, x: usize) -> Result<(), Error> {
check_invariants!(self);
if self.wcap() < x {
self.rewind();
}
if self.wcap() < x {
return Err(Error::NotEnoughSpace);
} else {
@@ -253,6 +256,9 @@ impl SlideBuf {
pub fn ___write_buf___(&mut self, n: usize) -> Result<&mut [u8], Error> {
check_invariants!(self);
self.rewind_if_needed(n);
if self.wcap() < n {
self.rewind();
}
if self.wcap() < n {
Err(Error::NotEnoughSpace)
} else {
@@ -283,6 +289,9 @@ impl SlideBuf {
pub fn available_writable_area(&mut self, need_min: usize) -> Result<&mut [u8], Error> {
check_invariants!(self);
self.rewind_if_needed(need_min);
if self.wcap() < need_min {
self.rewind();
}
if self.wcap() < need_min {
Err(Error::NotEnoughSpace)
} else {
@@ -294,6 +303,9 @@ impl SlideBuf {
pub fn put_slice(&mut self, buf: &[u8]) -> Result<(), Error> {
check_invariants!(self);
self.rewind_if_needed(buf.len());
if self.wcap() < buf.len() {
self.rewind();
}
if self.wcap() < buf.len() {
return Err(Error::NotEnoughSpace);
} else {
@@ -308,6 +320,9 @@ impl SlideBuf {
type T = u8;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {
@@ -322,6 +337,9 @@ impl SlideBuf {
type T = u16;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {
@@ -336,6 +354,9 @@ impl SlideBuf {
type T = u32;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {
@@ -350,6 +371,9 @@ impl SlideBuf {
type T = u64;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {
@@ -364,6 +388,9 @@ impl SlideBuf {
type T = f32;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {
@@ -378,6 +405,9 @@ impl SlideBuf {
type T = f64;
const TS: usize = std::mem::size_of::<T>();
self.rewind_if_needed(TS);
if self.wcap() < TS {
self.rewind();
}
if self.wcap() < TS {
return Err(Error::NotEnoughSpace);
} else {

9
streams/src/tcprawclient.rs
View File

@@ -20,6 +20,7 @@ use netpod::{Node, PerfOpts};
use std::pin::Pin;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tracing::Instrument;
pub async fn x_processed_event_blobs_stream_from_node(
query: PlainEventsQuery,
@@ -43,6 +44,7 @@ pub async fn x_processed_event_blobs_stream_from_node(
netout.flush().await?;
netout.forget();
let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let frames = Box::pin(frames) as _;
let items = EventsFromFrames::new(frames);
Ok(Box::pin(items))
}
@@ -71,8 +73,11 @@ where
netout.flush().await?;
netout.forget();
// TODO for images, we need larger buffer capacity
let frames = InMemoryFrameAsyncReadStream::new(netin, 1024 * 1024 * 2);
let stream = EventsFromFrames::<_, T>::new(frames);
let frames = InMemoryFrameAsyncReadStream::new(netin, 1024 * 1024 * 2)
//.instrument(netpod::log::span!(netpod::log::Level::TRACE, "InMemRd"))
;
let frames = Box::pin(frames) as _;
let stream = EventsFromFrames::<T>::new(frames);
streams.push(Box::pin(stream) as _);
}
Ok(streams)