daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor more

Browse Source
This commit is contained in:
Dominik Werder
2021-05-21 12:04:59 +02:00
parent b056811800
commit fec9a84567
6 changed files with 80 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
disk/src/binned.rs
View File

@@ -391,10 +391,10 @@ pub async fn binned_json(node_config: &NodeConfigCached, query: &BinnedQuery) ->
Ok(serde_json::to_value(ret)?)
}
pub trait BinnedStreamKind {
pub trait BinnedStreamKind: Clone + Unpin + Send + Sync + 'static {
type BinnedStreamItem: MakeBytesFrame;
type BinnedStreamType: Stream + Send + 'static;
type Dummy: Default + Unpin;
type Dummy: Default + Unpin + Send;
fn new_binned_from_prebinned(
&self,
@@ -413,6 +413,7 @@ pub trait BinnedStreamKind {
) -> Result<Self::BinnedStreamType, Error>;
}
#[derive(Clone)]
pub struct BinnedStreamKindScalar {}
pub struct BinnedStreamKindWave {}
@@ -449,7 +450,7 @@ impl BinnedStreamKind for BinnedStreamKindScalar {
query.cache_usage().clone(),
node_config,
query.disk_stats_every().clone(),
self,
self.clone(),
)?;
Ok(BinnedStream::new(Box::pin(s))?)
}

11
disk/src/binnedstream.rs
View File

@@ -32,7 +32,7 @@ where
cache_usage: CacheUsage,
node_config: &NodeConfigCached,
disk_stats_every: ByteSize,
stream_kind: &BK,
stream_kind: BK,
) -> Result<Self, Error> {
let patches: Vec<_> = patch_it.collect();
let mut sp = String::new();
@@ -55,15 +55,15 @@ where
cache_usage.clone(),
disk_stats_every.clone(),
);
let s: Pin<Box<dyn Stream<Item = _> + Send>> =
match PreBinnedScalarValueFetchedStream::new(&query, &node_config) {
let ret: Pin<Box<dyn Stream<Item = _> + Send>> =
match PreBinnedScalarValueFetchedStream::new(&query, &node_config, &stream_kind) {
Ok(k) => Box::pin(k),
Err(e) => {
error!("error from PreBinnedValueFetchedStream::new {:?}", e);
Box::pin(futures_util::stream::iter(vec![Err(e)]))
}
};
s
ret
}
})
.flatten()
@@ -99,9 +99,10 @@ where
}
})
.into_binned_t(range);
let mm = BK::Dummy::default();
Ok(Self {
inp: Box::pin(inp),
_marker: BK::Dummy::default(),
_marker: mm,
})
}
}

11
disk/src/cache.rs
View File

@@ -1,6 +1,7 @@
use crate::agg::eventbatch::MinMaxAvgScalarEventBatchStreamItem;
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::StreamItem;
use crate::binned::BinnedStreamKind;
use crate::cache::pbv::PreBinnedValueByteStream;
use crate::cache::pbvfs::PreBinnedScalarItem;
use crate::merge::MergedMinMaxAvgScalarStream;
@@ -231,10 +232,14 @@ fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Err
// NOTE This answers a request for a single valid pre-binned patch.
// A user must first make sure that the grid spec is valid, and that this node is responsible for it.
// Otherwise it is an error.
pub fn pre_binned_bytes_for_http(
pub fn pre_binned_bytes_for_http<BK>(
node_config: &NodeConfigCached,
query: &PreBinnedQuery,
) -> Result<PreBinnedValueByteStream, Error> {
stream_kind: BK,
) -> Result<PreBinnedValueByteStream<BK>, Error>
where
BK: BinnedStreamKind,
{
if query.channel.backend != node_config.node.backend {
let err = Error::with_msg(format!(
"backend mismatch node: {} requested: {}",
@@ -249,7 +254,7 @@ pub fn pre_binned_bytes_for_http(
node_config.ix, patch_node_ix
)))
} else {
let ret = super::cache::pbv::pre_binned_value_byte_stream_new(query, node_config);
let ret = super::cache::pbv::pre_binned_value_byte_stream_new(query, node_config, stream_kind);
Ok(ret)
}
}

47
disk/src/cache/pbv.rs vendored
View File

@@ -1,6 +1,7 @@
use crate::agg::binnedt::IntoBinnedT;
use crate::agg::scalarbinbatch::{MinMaxAvgScalarBinBatch, MinMaxAvgScalarBinBatchStreamItem};
use crate::agg::streams::StreamItem;
use crate::binned::BinnedStreamKind;
use crate::cache::pbvfs::{PreBinnedScalarItem, PreBinnedScalarValueFetchedStream};
use crate::cache::{CacheFileDesc, MergedFromRemotes, PreBinnedQuery};
use crate::frame::makeframe::make_frame;
@@ -18,22 +19,32 @@ use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
pub type PreBinnedValueByteStream = SCC<PreBinnedValueByteStreamInner>;
pub type PreBinnedValueByteStream<BK> = SCC<PreBinnedValueByteStreamInner<BK>>;
pub struct PreBinnedValueByteStreamInner {
inp: PreBinnedValueStream,
pub struct PreBinnedValueByteStreamInner<BK>
where
BK: BinnedStreamKind,
{
inp: PreBinnedValueStream<BK>,
}
pub fn pre_binned_value_byte_stream_new(
pub fn pre_binned_value_byte_stream_new<BK>(
query: &PreBinnedQuery,
node_config: &NodeConfigCached,
) -> PreBinnedValueByteStream {
let s1 = PreBinnedValueStream::new(query.clone(), node_config);
stream_kind: BK,
) -> PreBinnedValueByteStream<BK>
where
BK: BinnedStreamKind + Unpin,
{
let s1 = PreBinnedValueStream::new(query.clone(), node_config, stream_kind);
let s2 = PreBinnedValueByteStreamInner { inp: s1 };
SCC::new(s2)
}
impl Stream for PreBinnedValueByteStreamInner {
impl<BK> Stream for PreBinnedValueByteStreamInner<BK>
where
BK: BinnedStreamKind + Unpin,
{
type Item = Result<Bytes, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
@@ -49,7 +60,10 @@ impl Stream for PreBinnedValueByteStreamInner {
}
}
pub struct PreBinnedValueStream {
pub struct PreBinnedValueStream<BK>
where
BK: BinnedStreamKind,
{
query: PreBinnedQuery,
node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>,
@@ -65,10 +79,14 @@ pub struct PreBinnedValueStream {
values: MinMaxAvgScalarBinBatch,
write_fut: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
read_cache_fut: Option<Pin<Box<dyn Future<Output = Result<StreamItem<PreBinnedScalarItem>, Error>> + Send>>>,
stream_kind: BK,
}
impl PreBinnedValueStream {
pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached) -> Self {
impl<BK> PreBinnedValueStream<BK>
where
BK: BinnedStreamKind,
{
pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: BK) -> Self {
Self {
query,
node_config: node_config.clone(),
@@ -85,6 +103,7 @@ impl PreBinnedValueStream {
values: MinMaxAvgScalarBinBatch::empty(),
write_fut: None,
read_cache_fut: None,
stream_kind,
}
}
@@ -162,6 +181,7 @@ impl PreBinnedValueStream {
.map({
let q2 = self.query.clone();
let disk_stats_every = self.query.disk_stats_every.clone();
let stream_kind = self.stream_kind.clone();
move |patch| {
let query = PreBinnedQuery {
patch,
@@ -170,7 +190,7 @@ impl PreBinnedValueStream {
cache_usage: q2.cache_usage.clone(),
disk_stats_every: disk_stats_every.clone(),
};
PreBinnedScalarValueFetchedStream::new(&query, &node_config)
PreBinnedScalarValueFetchedStream::new(&query, &node_config, &stream_kind)
}
})
.map(|k| {
@@ -199,7 +219,10 @@ impl PreBinnedValueStream {
}
}
impl Stream for PreBinnedValueStream {
impl<BK> Stream for PreBinnedValueStream<BK>
where
BK: BinnedStreamKind,
{
// TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;

21
disk/src/cache/pbvfs.rs vendored
View File

@@ -1,5 +1,6 @@
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::StreamItem;
use crate::binned::BinnedStreamKind;
use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead, PreBinnedQuery};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::decode_frame;
@@ -7,23 +8,29 @@ use err::Error;
use futures_core::Stream;
use futures_util::{pin_mut, FutureExt};
use http::StatusCode;
#[allow(unused_imports)]
use netpod::log::*;
use netpod::{NodeConfigCached, PerfOpts};
use serde::{Deserialize, Serialize};
use std::pin::Pin;
use std::task::{Context, Poll};
pub struct PreBinnedScalarValueFetchedStream {
pub struct PreBinnedScalarValueFetchedStream<BK>
where
BK: BinnedStreamKind,
{
uri: http::Uri,
resfut: Option<hyper::client::ResponseFuture>,
res: Option<InMemoryFrameAsyncReadStream<HttpBodyAsAsyncRead>>,
errored: bool,
completed: bool,
stream_kind: BK,
}
impl PreBinnedScalarValueFetchedStream {
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
impl<BK> PreBinnedScalarValueFetchedStream<BK>
where
BK: BinnedStreamKind,
{
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: &BK) -> Result<Self, Error> {
let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize];
let uri: hyper::Uri = format!(
@@ -39,6 +46,7 @@ impl PreBinnedScalarValueFetchedStream {
res: None,
errored: false,
completed: false,
stream_kind: stream_kind.clone(),
};
Ok(ret)
}
@@ -50,7 +58,10 @@ pub enum PreBinnedScalarItem {
RangeComplete,
}
impl Stream for PreBinnedScalarValueFetchedStream {
impl<BK> Stream for PreBinnedScalarValueFetchedStream<BK>
where
BK: BinnedStreamKind,
{
type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {

16
httpret/src/lib.rs
View File

@@ -1,4 +1,5 @@
use bytes::Bytes;
use disk::binned::BinnedStreamKindScalar;
use disk::cache::{BinnedQuery, PreBinnedQuery};
use disk::raw::conn::raw_service;
use err::Error;
@@ -271,17 +272,22 @@ async fn binned_json(query: BinnedQuery, node_config: &NodeConfigCached) -> Resu
async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
let q = PreBinnedQuery::from_request(&head)?;
let desc = format!("pre-b-{}", q.patch().bin_t_len() / 1000000000);
let query = PreBinnedQuery::from_request(&head)?;
let desc = format!("pre-b-{}", query.patch().bin_t_len() / 1000000000);
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str());
// TODO factor from the inner scopes the fetch of the channel entry so that I can
// provide the stream_kind here:
let stream_kind = BinnedStreamKindScalar::new();
span1.in_scope(|| {
let ret = match disk::cache::pre_binned_bytes_for_http(node_config, &q) {
let ret = match disk::cache::pre_binned_bytes_for_http(node_config, &query, stream_kind) {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(
s,
format!(
"pre-b-{}-p-{}",
q.patch().bin_t_len() / 1000000000,
q.patch().patch_beg() / 1000000000,
query.patch().bin_t_len() / 1000000000,
query.patch().patch_beg() / 1000000000,
),
))?,
Err(e) => {