daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on adding stats to the streams

Browse Source
This commit is contained in:
Dominik Werder
2021-05-01 18:00:05 +02:00
parent 0f9408e9f8
commit 0eb6b3364d
15 changed files with 275 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
disk/src/cache/pbv.rs vendored
View File

@@ -1,6 +1,5 @@
use crate::agg::binnedt::IntoBinnedT;
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::cache::pbvfs::{PreBinnedHttpFrame, PreBinnedValueFetchedStream};
use crate::cache::pbvfs::{PreBinnedFrame, PreBinnedItem, PreBinnedValueFetchedStream};
use crate::cache::{node_ix_for_patch, MergedFromRemotes};
use crate::frame::makeframe::make_frame;
use crate::raw::EventsQuery;
@@ -11,7 +10,7 @@ use futures_util::{FutureExt, StreamExt};
use netpod::log::*;
use netpod::streamext::SCC;
use netpod::{
AggKind, BinnedRange, Channel, NanoRange, NodeConfig, PreBinnedPatchCoord, PreBinnedPatchIterator,
AggKind, BinnedRange, Channel, NanoRange, NodeConfigCached, PreBinnedPatchCoord, PreBinnedPatchIterator,
PreBinnedPatchRange,
};
use std::future::Future;
@@ -28,7 +27,7 @@ pub fn pre_binned_value_byte_stream_new(
patch: PreBinnedPatchCoord,
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfig,
node_config: &NodeConfigCached,
) -> PreBinnedValueByteStream {
let s1 = PreBinnedValueStream::new(patch, channel, agg_kind, node_config);
let s2 = PreBinnedValueByteStreamInner { inp: s1 };
@@ -41,7 +40,7 @@ impl Stream for PreBinnedValueByteStreamInner {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => match make_frame::<PreBinnedHttpFrame>(&item) {
Ready(Some(item)) => match make_frame::<PreBinnedFrame>(&item) {
Ok(buf) => Ready(Some(Ok(buf.freeze()))),
Err(e) => Ready(Some(Err(e.into()))),
},
@@ -55,9 +54,9 @@ pub struct PreBinnedValueStream {
patch_coord: PreBinnedPatchCoord,
channel: Channel,
agg_kind: AggKind,
node_config: NodeConfig,
node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = Result<MinMaxAvgScalarBinBatch, Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = PreBinnedFrame> + Send>>>,
errored: bool,
completed: bool,
}
@@ -67,10 +66,10 @@ impl PreBinnedValueStream {
patch_coord: PreBinnedPatchCoord,
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfig,
node_config: &NodeConfigCached,
) -> Self {
// TODO check that we are the correct node.
let _node_ix = node_ix_for_patch(&patch_coord, &channel, &node_config.cluster);
let _node_ix = node_ix_for_patch(&patch_coord, &channel, &node_config.node_config.cluster);
Self {
patch_coord,
channel,
@@ -129,7 +128,7 @@ impl PreBinnedValueStream {
error!("try_setup_fetch_prebinned_higher_res apply all requested transformations and T-binning");
let count = self.patch_coord.patch_t_len() / self.patch_coord.bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap();
let s1 = MergedFromRemotes::new(evq, self.node_config.cluster.clone());
let s1 = MergedFromRemotes::new(evq, self.node_config.node_config.cluster.clone());
let s2 = s1
.map(|k| {
if k.is_err() {
@@ -139,7 +138,11 @@ impl PreBinnedValueStream {
}
k
})
.into_binned_t(range);
.into_binned_t(range)
.map(|k| match k {
Ok(k) => PreBinnedFrame(Ok(PreBinnedItem::Batch(k))),
Err(e) => PreBinnedFrame(Err(e)),
});
self.fut2 = Some(Box::pin(s2));
}
}
@@ -148,7 +151,7 @@ impl PreBinnedValueStream {
impl Stream for PreBinnedValueStream {
// TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
type Item = Result<MinMaxAvgScalarBinBatch, Error>;
type Item = PreBinnedFrame;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -162,31 +165,33 @@ impl Stream for PreBinnedValueStream {
'outer: loop {
break if let Some(fut) = self.fut2.as_mut() {
match fut.poll_next_unpin(cx) {
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
Ready(Some(Err(e))) => {
self.errored = true;
Ready(Some(Err(e)))
}
Ready(Some(k)) => match k.0 {
Ok(k) => Ready(Some(PreBinnedFrame(Ok(k)))),
Err(e) => {
self.errored = true;
Ready(Some(PreBinnedFrame(Err(e))))
}
},
Ready(None) => Ready(None),
Pending => Pending,
}
} else if let Some(fut) = self.open_check_local_file.as_mut() {
match fut.poll_unpin(cx) {
Ready(Ok(_file)) => {
let e = Err(Error::with_msg(format!("TODO use the cached data from file")));
let e = PreBinnedFrame(e);
self.errored = true;
Ready(Some(Err(Error::with_msg(format!(
"TODO use the cached data from file"
)))))
Ready(Some(e))
}
Ready(Err(e)) => match e.kind() {
std::io::ErrorKind::NotFound => {
error!("TODO LOCAL CACHE FILE NOT FOUND");
self.try_setup_fetch_prebinned_higher_res();
if self.fut2.is_none() {
let e = Err(Error::with_msg(format!("try_setup_fetch_prebinned_higher_res failed")));
let e = PreBinnedFrame(e);
self.errored = true;
Ready(Some(Err(Error::with_msg(format!(
"try_setup_fetch_prebinned_higher_res failed"
)))))
Ready(Some(e))
} else {
continue 'outer;
}
@@ -194,7 +199,8 @@ impl Stream for PreBinnedValueStream {
_ => {
error!("File I/O error: {:?}", e);
self.errored = true;
Ready(Some(Err(e.into())))
let e = PreBinnedFrame(Err(e.into()));
Ready(Some(e))
}
},
Pending => Pending,

63
disk/src/cache/pbvfs.rs vendored
View File

@@ -7,7 +7,8 @@ use futures_core::Stream;
use futures_util::{pin_mut, FutureExt};
#[allow(unused_imports)]
use netpod::log::*;
use netpod::{AggKind, Channel, NodeConfig, PreBinnedPatchCoord};
use netpod::{AggKind, Channel, NodeConfigCached, PreBinnedPatchCoord};
use serde::{Deserialize, Serialize};
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -24,10 +25,10 @@ impl PreBinnedValueFetchedStream {
patch_coord: PreBinnedPatchCoord,
channel: Channel,
agg_kind: AggKind,
node_config: &NodeConfig,
node_config: &NodeConfigCached,
) -> Self {
let nodeix = node_ix_for_patch(&patch_coord, &channel, &node_config.cluster);
let node = &node_config.cluster.nodes[nodeix as usize];
let nodeix = node_ix_for_patch(&patch_coord, &channel, &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize];
warn!("TODO defining property of a PreBinnedPatchCoord? patchlen + ix? binsize + patchix? binsize + patchsize + patchix?");
// TODO encapsulate uri creation, how to express aggregation kind?
let uri: hyper::Uri = format!(
@@ -52,13 +53,26 @@ impl PreBinnedValueFetchedStream {
}
}
// TODO use a newtype here to use a different FRAME_TYPE_ID compared to
// impl FrameType for BinnedBytesForHttpStreamFrame
pub type PreBinnedHttpFrame = Result<MinMaxAvgScalarBinBatch, Error>;
#[derive(Serialize, Deserialize)]
pub struct PreBinnedFrame(pub Result<PreBinnedItem, Error>);
impl<T> From<T> for PreBinnedFrame
where
T: Into<Error>,
{
fn from(k: T) -> Self {
PreBinnedFrame(Err(k.into()))
}
}
#[derive(Serialize, Deserialize)]
pub enum PreBinnedItem {
Batch(MinMaxAvgScalarBinBatch),
}
impl Stream for PreBinnedValueFetchedStream {
// TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
type Item = PreBinnedHttpFrame;
type Item = PreBinnedFrame;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -73,16 +87,19 @@ impl Stream for PreBinnedValueFetchedStream {
break if let Some(res) = self.res.as_mut() {
pin_mut!(res);
match res.poll_next(cx) {
Ready(Some(Ok(frame))) => match decode_frame::<PreBinnedHttpFrame>(&frame) {
Ok(item) => Ready(Some(item)),
Ready(Some(Ok(frame))) => match decode_frame::<PreBinnedFrame>(&frame) {
Ok(item) => match item.0 {
Ok(item) => Ready(Some(PreBinnedFrame(Ok(item)))),
Err(e) => Ready(Some(PreBinnedFrame(Err(e)))),
},
Err(e) => {
self.errored = true;
Ready(Some(Err(e.into())))
Ready(Some(e.into()))
}
},
Ready(Some(Err(e))) => {
self.errored = true;
Ready(Some(Err(e.into())))
Ready(Some(e.into()))
}
Ready(None) => {
self.completed = true;
@@ -103,20 +120,28 @@ impl Stream for PreBinnedValueFetchedStream {
Err(e) => {
error!("PreBinnedValueStream error in stream {:?}", e);
self.errored = true;
Ready(Some(Err(e.into())))
Ready(Some(PreBinnedFrame(Err(e.into()))))
}
},
Pending => Pending,
}
} else {
let req = hyper::Request::builder()
match hyper::Request::builder()
.method(http::Method::GET)
.uri(&self.uri)
.body(hyper::Body::empty())?;
let client = hyper::Client::new();
info!("PreBinnedValueFetchedStream START REQUEST FOR {:?}", req);
self.resfut = Some(client.request(req));
continue 'outer;
.body(hyper::Body::empty())
{
Ok(req) => {
let client = hyper::Client::new();
info!("PreBinnedValueFetchedStream START REQUEST FOR {:?}", req);
self.resfut = Some(client.request(req));
continue 'outer;
}
Err(e) => {
self.errored = true;
Ready(Some(e.into()))
}
}
};
}
}