daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on factor stream kind

Browse Source
This commit is contained in:
Dominik Werder
2021-05-21 11:40:35 +02:00
parent a959250af9
commit b056811800
9 changed files with 64 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
disk/src/cache/pbv.rs vendored
View File

@@ -1,7 +1,7 @@
use crate::agg::binnedt::IntoBinnedT;
use crate::agg::scalarbinbatch::{MinMaxAvgScalarBinBatch, MinMaxAvgScalarBinBatchStreamItem};
use crate::agg::streams::StreamItem;
use crate::cache::pbvfs::{PreBinnedItem, PreBinnedValueFetchedStream};
use crate::cache::pbvfs::{PreBinnedScalarItem, PreBinnedScalarValueFetchedStream};
use crate::cache::{CacheFileDesc, MergedFromRemotes, PreBinnedQuery};
use crate::frame::makeframe::make_frame;
use crate::raw::EventsQuery;
@@ -39,7 +39,7 @@ impl Stream for PreBinnedValueByteStreamInner {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => match make_frame::<Result<StreamItem<PreBinnedItem>, Error>>(&item) {
Ready(Some(item)) => match make_frame::<Result<StreamItem<PreBinnedScalarItem>, Error>>(&item) {
Ok(buf) => Ready(Some(Ok(buf.freeze()))),
Err(e) => Ready(Some(Err(e.into()))),
},
@@ -53,7 +53,7 @@ pub struct PreBinnedValueStream {
query: PreBinnedQuery,
node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<tokio::fs::File, std::io::Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = Result<StreamItem<PreBinnedItem>, Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = Result<StreamItem<PreBinnedScalarItem>, Error>> + Send>>>,
read_from_cache: bool,
cache_written: bool,
data_complete: bool,
@@ -64,7 +64,7 @@ pub struct PreBinnedValueStream {
streamlog: Streamlog,
values: MinMaxAvgScalarBinBatch,
write_fut: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
read_cache_fut: Option<Pin<Box<dyn Future<Output = Result<StreamItem<PreBinnedItem>, Error>> + Send>>>,
read_cache_fut: Option<Pin<Box<dyn Future<Output = Result<StreamItem<PreBinnedScalarItem>, Error>> + Send>>>,
}
impl PreBinnedValueStream {
@@ -120,10 +120,10 @@ impl PreBinnedValueStream {
StreamItem::Stats(item) => Ok(StreamItem::Stats(item)),
StreamItem::DataItem(item) => match item {
MinMaxAvgScalarBinBatchStreamItem::RangeComplete => {
Ok(StreamItem::DataItem(PreBinnedItem::RangeComplete))
Ok(StreamItem::DataItem(PreBinnedScalarItem::RangeComplete))
}
MinMaxAvgScalarBinBatchStreamItem::Values(item) => {
Ok(StreamItem::DataItem(PreBinnedItem::Batch(item)))
Ok(StreamItem::DataItem(PreBinnedScalarItem::Batch(item)))
}
},
},
@@ -170,7 +170,7 @@ impl PreBinnedValueStream {
cache_usage: q2.cache_usage.clone(),
disk_stats_every: disk_stats_every.clone(),
};
PreBinnedValueFetchedStream::new(&query, &node_config)
PreBinnedScalarValueFetchedStream::new(&query, &node_config)
}
})
.map(|k| {
@@ -201,7 +201,7 @@ impl PreBinnedValueStream {
impl Stream for PreBinnedValueStream {
// TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
type Item = Result<StreamItem<PreBinnedItem>, Error>;
type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -256,7 +256,7 @@ impl Stream for PreBinnedValueStream {
if self.cache_written {
if self.range_complete_observed {
self.range_complete_emitted = true;
Ready(Some(Ok(StreamItem::DataItem(PreBinnedItem::RangeComplete))))
Ready(Some(Ok(StreamItem::DataItem(PreBinnedScalarItem::RangeComplete))))
} else {
self.completed = true;
Ready(None)
@@ -297,18 +297,18 @@ impl Stream for PreBinnedValueStream {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(item) => match item {
PreBinnedItem::RangeComplete => {
PreBinnedScalarItem::RangeComplete => {
self.range_complete_observed = true;
continue 'outer;
}
PreBinnedItem::Batch(batch) => {
PreBinnedScalarItem::Batch(batch) => {
self.values.ts1s.extend(batch.ts1s.iter());
self.values.ts2s.extend(batch.ts2s.iter());
self.values.counts.extend(batch.counts.iter());
self.values.mins.extend(batch.mins.iter());
self.values.maxs.extend(batch.maxs.iter());
self.values.avgs.extend(batch.avgs.iter());
Ready(Some(Ok(StreamItem::DataItem(PreBinnedItem::Batch(batch)))))
Ready(Some(Ok(StreamItem::DataItem(PreBinnedScalarItem::Batch(batch)))))
}
},
},

27
disk/src/cache/pbvfs.rs vendored
View File

@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
use std::pin::Pin;
use std::task::{Context, Poll};
pub struct PreBinnedValueFetchedStream {
pub struct PreBinnedScalarValueFetchedStream {
uri: http::Uri,
resfut: Option<hyper::client::ResponseFuture>,
res: Option<InMemoryFrameAsyncReadStream<HttpBodyAsAsyncRead>>,
@@ -22,7 +22,7 @@ pub struct PreBinnedValueFetchedStream {
completed: bool,
}
impl PreBinnedValueFetchedStream {
impl PreBinnedScalarValueFetchedStream {
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize];
@@ -45,33 +45,30 @@ impl PreBinnedValueFetchedStream {
}
#[derive(Debug, Serialize, Deserialize)]
pub enum PreBinnedItem {
pub enum PreBinnedScalarItem {
Batch(MinMaxAvgScalarBinBatch),
RangeComplete,
}
impl Stream for PreBinnedValueFetchedStream {
// TODO need this generic for scalar and array (when wave is not binned down to a single scalar point)
type Item = Result<StreamItem<PreBinnedItem>, Error>;
impl Stream for PreBinnedScalarValueFetchedStream {
type Item = Result<StreamItem<PreBinnedScalarItem>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
if self.completed {
panic!("poll_next on completed");
}
if self.errored {
self.completed = true;
return Ready(None);
}
'outer: loop {
break if let Some(res) = self.res.as_mut() {
break if self.completed {
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
return Ready(None);
} else if let Some(res) = self.res.as_mut() {
pin_mut!(res);
match res.poll_next(cx) {
Ready(Some(Ok(item))) => match item {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(item) => {
match decode_frame::<Result<StreamItem<PreBinnedItem>, Error>>(&item) {
match decode_frame::<Result<StreamItem<PreBinnedScalarItem>, Error>>(&item) {
Ok(Ok(item)) => Ready(Some(Ok(item))),
Ok(Err(e)) => {
self.errored = true;