daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2021-05-07 08:25:13 +02:00
parent a89e1cfc50
commit db93ae1545
10 changed files with 70 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
disk/src/cache.rs
View File

@@ -15,8 +15,8 @@ use futures_core::Stream;
use futures_util::{pin_mut, StreamExt};
use hyper::Response;
use netpod::{
AggKind, BinnedRange, Channel, Cluster, NanoRange, NodeConfigCached, PreBinnedPatchCoord, PreBinnedPatchIterator,
PreBinnedPatchRange, ToNanos,
AggKind, BinnedRange, Channel, Cluster, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchCoord,
PreBinnedPatchIterator, PreBinnedPatchRange, ToNanos,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
@@ -174,6 +174,7 @@ pub async fn binned_bytes_for_http(
let range = BinnedRange::covering_range(range.clone(), query.bin_count).ok_or(Error::with_msg(format!(
"binned_bytes_for_http BinnedRange::covering_range returned None"
)))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
match PreBinnedPatchRange::covering_range(query.range.clone(), query.bin_count) {
Some(pre_range) => {
info!("binned_bytes_for_http found pre_range: {:?}", pre_range);
@@ -206,7 +207,7 @@ pub async fn binned_bytes_for_http(
agg_kind: query.agg_kind.clone(),
};
// TODO do I need to set up more transformations or binning to deliver the requested data?
let s1 = MergedFromRemotes::new(evq, node_config.node_config.cluster.clone());
let s1 = MergedFromRemotes::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s1 = s1.into_binned_t(range);
/*let s1 = s1.map(|k| {
use super::agg::scalarbinbatch::MinMaxAvgScalarBinBatchStreamItem::*;
@@ -372,10 +373,10 @@ pub struct MergedFromRemotes {
}
impl MergedFromRemotes {
pub fn new(evq: EventsQuery, cluster: Cluster) -> Self {
pub fn new(evq: EventsQuery, perf_opts: PerfOpts, cluster: Cluster) -> Self {
let mut tcp_establish_futs = vec![];
for node in &cluster.nodes {
let f = super::raw::x_processed_stream_from_node(evq.clone(), node.clone());
let f = super::raw::x_processed_stream_from_node(evq.clone(), perf_opts.clone(), node.clone());
let f: T002 = Box::pin(f);
tcp_establish_futs.push(f);
}

20
disk/src/cache/pbv.rs vendored
View File

@@ -11,8 +11,8 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use netpod::log::*;
use netpod::streamext::SCC;
use netpod::{BinnedRange, NodeConfigCached, PreBinnedPatchIterator, PreBinnedPatchRange};
use std::future::{ready, Future};
use netpod::{BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange};
use std::future::Future;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -105,7 +105,8 @@ impl PreBinnedValueStream {
// TODO do I need to set up more transformations or binning to deliver the requested data?
let count = self.query.patch.patch_t_len() / self.query.patch.bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap();
let s1 = MergedFromRemotes::new(evq, self.node_config.node_config.cluster.clone());
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s1 = MergedFromRemotes::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let s1 = s1.into_binned_t(range);
let s1 = s1.map(|k| {
use MinMaxAvgScalarBinBatchStreamItem::*;
@@ -161,13 +162,12 @@ impl PreBinnedValueStream {
PreBinnedValueFetchedStream::new(&query, &node_config)
}
})
.filter_map(|k| match k {
Ok(k) => ready(Some(k)),
Err(e) => {
// TODO Reconsider error handling here:
error!("{:?}", e);
ready(None)
}
.map(|k| {
let s: Pin<Box<dyn Stream<Item = _> + Send>> = match k {
Ok(k) => Box::pin(k),
Err(e) => Box::pin(futures_util::stream::iter(vec![Err(e)])),
};
s
})
.flatten();
self.fut2 = Some(Box::pin(s));

5
disk/src/cache/pbvfs.rs vendored
View File

@@ -9,7 +9,7 @@ use futures_util::{pin_mut, FutureExt};
use http::StatusCode;
#[allow(unused_imports)]
use netpod::log::*;
use netpod::{EventDataReadStats, NodeConfigCached};
use netpod::{EventDataReadStats, NodeConfigCached, PerfOpts};
use serde::{Deserialize, Serialize};
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -104,8 +104,9 @@ impl Stream for PreBinnedValueFetchedStream {
Ready(res) => match res {
Ok(res) => {
if res.status() == StatusCode::OK {
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1);
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
self.res = Some(s2);
continue 'outer;
} else {

6
disk/src/frame/inmem.rs
View File

@@ -31,13 +31,11 @@ impl<T> InMemoryFrameAsyncReadStream<T>
where
T: AsyncRead + Unpin,
{
pub fn new(inp: T) -> Self {
// TODO make capacity adjustable.
let bufcap = 512;
pub fn new(inp: T, bufcap: usize) -> Self {
let mut t = Self {
inp,
buf: BytesMut::new(),
bufcap: bufcap,
bufcap,
wp: 0,
tryparse: false,
errored: false,

5
disk/src/raw.rs
View File

@@ -11,7 +11,7 @@ use crate::frame::makeframe::{make_frame, make_term_frame};
use crate::raw::bffr::MinMaxAvgScalarEventBatchStreamFromFrames;
use err::Error;
use futures_core::Stream;
use netpod::{AggKind, Channel, NanoRange, Node};
use netpod::{AggKind, Channel, NanoRange, Node, PerfOpts};
use serde::{Deserialize, Serialize};
use std::pin::Pin;
use tokio::io::AsyncWriteExt;
@@ -37,6 +37,7 @@ pub struct EventQueryJsonStringFrame(String);
pub async fn x_processed_stream_from_node(
query: EventsQuery,
perf_opts: PerfOpts,
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Result<MinMaxAvgScalarEventBatchStreamItem, Error>> + Send>>, Error> {
let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
@@ -48,7 +49,7 @@ pub async fn x_processed_stream_from_node(
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
let frames = InMemoryFrameAsyncReadStream::new(netin);
let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let items = MinMaxAvgScalarEventBatchStreamFromFrames::new(frames);
Ok(Box::pin(items))
}

64
disk/src/raw/bffr.rs
View File

@@ -40,28 +40,22 @@ where
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
if self.completed {
panic!("MinMaxAvgScalarEventBatchStreamFromFrames poll_next on completed");
}
if self.errored {
self.completed = true;
return Ready(None);
}
loop {
break match self.inp.poll_next_unpin(cx) {
Ready(Some(Ok(frame))) => {
type ExpectedType = RawConnOut;
trace!(
"MinMaxAvgScalarEventBatchStreamFromFrames got full frame buf {}",
frame.buf().len()
);
match decode_frame::<ExpectedType>(&frame) {
Ok(item) => {
match item {
break if self.completed {
panic!("MinMaxAvgScalarEventBatchStreamFromFrames poll_next on completed");
} else if self.errored {
self.completed = true;
Ready(None)
} else {
match self.inp.poll_next_unpin(cx) {
Ready(Some(Ok(frame))) => {
type ExpectedType = RawConnOut;
match decode_frame::<ExpectedType>(&frame) {
Ok(item) => match item {
Ok(item) => {
match &item {
MinMaxAvgScalarEventBatchStreamItem::EventDataReadStats(stats) => {
info!("✒✒✒✒✒✒✒✒✒✒✒✒✒✒✒✒ MinMaxAvgScalarEventBatchStreamFromFrames stats {:?}", stats);
info!("✒✒ ✒✒ ✒✒ ✒✒ ✒✒ ✒✒ stats {:?}", stats);
}
_ => {
info!("✒ ✒ ✒ ✒ other kind")
@@ -73,27 +67,27 @@ where
self.errored = true;
Ready(Some(Err(e)))
}
},
Err(e) => {
error!(
"MinMaxAvgScalarEventBatchStreamFromFrames ~~~~~~~~ ERROR on frame payload {}",
frame.buf().len(),
);
self.errored = true;
Ready(Some(Err(e)))
}
}
Err(e) => {
error!(
"MinMaxAvgScalarEventBatchStreamFromFrames ~~~~~~~~ ERROR on frame payload {}",
frame.buf().len(),
);
self.errored = true;
Ready(Some(Err(e)))
}
}
Ready(Some(Err(e))) => {
self.errored = true;
Ready(Some(Err(e)))
}
Ready(None) => {
self.completed = true;
Ready(None)
}
Pending => Pending,
}
Ready(Some(Err(e))) => {
self.errored = true;
Ready(Some(Err(e)))
}
Ready(None) => {
self.completed = true;
Ready(None)
}
Pending => Pending,
};
}
}

5
disk/src/raw/conn.rs
View File

@@ -9,7 +9,7 @@ use crate::raw::{EventQueryJsonStringFrame, EventsQuery};
use err::Error;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{NodeConfigCached, Shape};
use netpod::{NodeConfigCached, PerfOpts, Shape};
use std::net::SocketAddr;
use tokio::io::AsyncWriteExt;
use tokio::net::tcp::OwnedWriteHalf;
@@ -85,7 +85,8 @@ async fn raw_conn_handler_inner_try(
) -> Result<(), ConnErr> {
let _ = addr;
let (netin, mut netout) = stream.into_split();
let mut h = InMemoryFrameAsyncReadStream::new(netin);
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let mut h = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
let mut frames = vec![];
while let Some(k) = h
.next()