daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename StreamKind

Browse Source
This commit is contained in:
Dominik Werder
2021-06-03 12:47:50 +02:00
parent ad62f125e3
commit c19c4997bb
15 changed files with 113 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
disk/src/agg.rs
View File

@@ -6,7 +6,7 @@ use super::eventchunker::EventFull;
use crate::agg::binnedt::AggregatableTdim;
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use bytes::BytesMut;
use err::Error;
use futures_core::Stream;
@@ -28,7 +28,7 @@ pub mod streams;
pub trait AggregatableXdim1Bin<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Output: AggregatableXdim1Bin<SK> + AggregatableTdim<SK>;
fn into_agg(self) -> Self::Output;
@@ -54,7 +54,7 @@ impl std::fmt::Debug for ValuesDim0 {
impl<SK> AggregatableXdim1Bin<SK> for ValuesDim1
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Output = MinMaxAvgScalarEventBatch;
@@ -151,7 +151,7 @@ impl std::fmt::Debug for ValuesDim1 {
impl<SK> AggregatableXdim1Bin<SK> for ValuesDim0
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Output = MinMaxAvgScalarEventBatch;

46
disk/src/agg/binnedt.rs
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
@@ -11,7 +11,7 @@ use std::task::{Context, Poll};
pub trait AggregatorTdim<SK>: Sized + Unpin
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type InputValue;
type OutputValue;
@@ -24,7 +24,7 @@ where
pub trait AggregatableTdim<SK>: Sized
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Aggregator: AggregatorTdim<SK>;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator;
@@ -32,16 +32,16 @@ where
pub trait IntoBinnedT<SK, S>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
{
fn into_binned_t(self, spec: BinnedRange) -> IntoBinnedTDefaultStream<SK, S>;
}
impl<SK, S> IntoBinnedT<SK, S> for S
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
{
fn into_binned_t(self, spec: BinnedRange) -> IntoBinnedTDefaultStream<SK, S> {
IntoBinnedTDefaultStream::new(self, spec)
@@ -50,36 +50,35 @@ where
pub struct IntoBinnedTDefaultStream<SK, S>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
{
inp: S,
aggtor: Option<<<SK as BinnedStreamKind>::XBinnedEvents as AggregatableTdim<SK>>::Aggregator>,
aggtor: Option<<<SK as StreamKind>::XBinnedEvents as AggregatableTdim<SK>>::Aggregator>,
spec: BinnedRange,
curbin: u32,
inp_completed: bool,
all_bins_emitted: bool,
range_complete_observed: bool,
range_complete_emitted: bool,
left:
Option<Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>>>>,
left: Option<Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>>>>,
errored: bool,
completed: bool,
tmp_agg_results: VecDeque<<SK as BinnedStreamKind>::TBinnedBins>,
tmp_agg_results: VecDeque<<SK as StreamKind>::TBinnedBins>,
_marker: std::marker::PhantomData<SK>,
}
impl<SK, S> IntoBinnedTDefaultStream<SK, S>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
{
pub fn new(inp: S, spec: BinnedRange) -> Self {
let range = spec.get_range(0);
Self {
inp,
aggtor: Some(
<<SK as BinnedStreamKind>::XBinnedEvents as AggregatableTdim<SK>>::aggregator_new_static(
<<SK as StreamKind>::XBinnedEvents as AggregatableTdim<SK>>::aggregator_new_static(
range.beg, range.end,
),
),
@@ -100,7 +99,7 @@ where
fn cur(
&mut self,
cx: &mut Context,
) -> Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>>> {
) -> Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>>> {
if let Some(cur) = self.left.take() {
cur
} else if self.inp_completed {
@@ -117,7 +116,7 @@ where
let _ret = self
.aggtor
.replace(
<<SK as BinnedStreamKind>::XBinnedEvents as AggregatableTdim<SK>>::aggregator_new_static(
<<SK as StreamKind>::XBinnedEvents as AggregatableTdim<SK>>::aggregator_new_static(
range.beg, range.end,
),
)
@@ -135,9 +134,8 @@ where
fn handle(
&mut self,
cur: Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>>>,
) -> Option<Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>>>>
{
cur: Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>>>,
) -> Option<Poll<Option<Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, Error>>>> {
use Poll::*;
match cur {
Ready(Some(Ok(item))) => match item {
@@ -205,10 +203,10 @@ where
impl<SK, S> Stream for IntoBinnedTDefaultStream<SK, S>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
{
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;

10
disk/src/agg/binnedx.rs
View File

@@ -1,6 +1,6 @@
use crate::agg::streams::StreamItem;
use crate::agg::AggregatableXdim1Bin;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
@@ -9,7 +9,7 @@ use std::task::{Context, Poll};
pub trait IntoBinnedXBins1<I, SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
Self: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin<SK>,
{
@@ -19,7 +19,7 @@ where
impl<S, I, SK> IntoBinnedXBins1<I, SK> for S
where
SK: BinnedStreamKind,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin<SK>,
{
@@ -35,7 +35,7 @@ where
pub struct IntoBinnedXBins1DefaultStream<S, I, SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin<SK>,
{
@@ -45,7 +45,7 @@ where
impl<S, I, SK> Stream for IntoBinnedXBins1DefaultStream<S, I, SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin<SK>,
{

8
disk/src/agg/eventbatch.rs
View File

@@ -2,7 +2,7 @@ use crate::agg::binnedt::{AggregatableTdim, AggregatorTdim};
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::{Appendable, StreamItem};
use crate::agg::AggregatableXdim1Bin;
use crate::binned::{BinnedStreamKind, MakeBytesFrame, RangeCompletableItem, RangeOverlapInfo};
use crate::binned::{MakeBytesFrame, RangeCompletableItem, RangeOverlapInfo, StreamKind};
use crate::frame::makeframe::make_frame;
use bytes::{BufMut, Bytes, BytesMut};
use err::Error;
@@ -103,7 +103,7 @@ impl std::fmt::Debug for MinMaxAvgScalarEventBatch {
impl<SK> AggregatableXdim1Bin<SK> for MinMaxAvgScalarEventBatch
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Output = MinMaxAvgScalarEventBatch;
fn into_agg(self) -> Self::Output {
@@ -113,7 +113,7 @@ where
impl<SK> AggregatableTdim<SK> for MinMaxAvgScalarEventBatch
where
SK: BinnedStreamKind,
SK: StreamKind,
{
//type Output = MinMaxAvgScalarBinBatch;
type Aggregator = MinMaxAvgScalarEventBatchAggregator;
@@ -174,7 +174,7 @@ impl MinMaxAvgScalarEventBatchAggregator {
impl<SK> AggregatorTdim<SK> for MinMaxAvgScalarEventBatchAggregator
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type InputValue = MinMaxAvgScalarEventBatch;
type OutputValue = MinMaxAvgScalarBinBatch;

8
disk/src/agg/scalarbinbatch.rs
View File

@@ -1,7 +1,7 @@
use crate::agg::binnedt::{AggregatableTdim, AggregatorTdim};
use crate::agg::streams::{Appendable, Bins, StreamItem};
use crate::agg::{AggregatableXdim1Bin, Fits, FitsInside};
use crate::binned::{BinnedStreamKind, MakeBytesFrame, RangeCompletableItem};
use crate::binned::{MakeBytesFrame, RangeCompletableItem, StreamKind};
use crate::frame::makeframe::make_frame;
use bytes::{BufMut, Bytes, BytesMut};
use err::Error;
@@ -187,7 +187,7 @@ impl MinMaxAvgScalarBinBatch {
impl<SK> AggregatableXdim1Bin<SK> for MinMaxAvgScalarBinBatch
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Output = MinMaxAvgScalarBinBatch;
fn into_agg(self) -> Self::Output {
@@ -197,7 +197,7 @@ where
impl<SK> AggregatableTdim<SK> for MinMaxAvgScalarBinBatch
where
SK: BinnedStreamKind,
SK: StreamKind,
{
//type Output = MinMaxAvgScalarBinBatch;
type Aggregator = MinMaxAvgScalarBinBatchAggregator;
@@ -239,7 +239,7 @@ impl MinMaxAvgScalarBinBatchAggregator {
impl<SK> AggregatorTdim<SK> for MinMaxAvgScalarBinBatchAggregator
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type InputValue = MinMaxAvgScalarBinBatch;
type OutputValue = MinMaxAvgScalarBinBatch;

8
disk/src/binned.rs
View File

@@ -501,7 +501,7 @@ pub trait XBinnedEvents<SK>:
+ PushableIndex
+ Appendable
where
SK: BinnedStreamKind,
SK: StreamKind,
{
fn frame_type() -> u32;
}
@@ -524,7 +524,7 @@ pub trait TBinnedBins:
impl<SK> XBinnedEvents<SK> for MinMaxAvgScalarEventBatch
where
SK: BinnedStreamKind,
SK: StreamKind,
{
fn frame_type() -> u32 {
<Result<StreamItem<RangeCompletableItem<Self>>, Error> as FrameType>::FRAME_TYPE_ID
@@ -537,7 +537,7 @@ impl TBinnedBins for MinMaxAvgScalarBinBatch {
}
}
pub trait BinnedStreamKind: Clone + Unpin + Send + Sync + 'static {
pub trait StreamKind: Clone + Unpin + Send + Sync + 'static {
type TBinnedStreamType: Stream<Item = Result<StreamItem<RangeCompletableItem<Self::TBinnedBins>>, Error>> + Send;
type XBinnedEvents: XBinnedEvents<Self>;
type TBinnedBins: TBinnedBins;
@@ -590,7 +590,7 @@ pub enum RangeCompletableItem<T> {
Data(T),
}
impl BinnedStreamKind for BinnedStreamKindScalar {
impl StreamKind for BinnedStreamKindScalar {
// TODO is this really needed?
type TBinnedStreamType = BoxedStream<Result<StreamItem<RangeCompletableItem<Self::TBinnedBins>>, Error>>;
type XBinnedEvents = MinMaxAvgScalarEventBatch;

4
disk/src/binned/scalar.rs
View File

@@ -1,4 +1,4 @@
use crate::binned::{BinnedStreamKind, BinnedStreamRes};
use crate::binned::{BinnedStreamRes, StreamKind};
use crate::binnedstream::BoxedStream;
use crate::cache::BinnedQuery;
use crate::raw::EventsQuery;
@@ -12,7 +12,7 @@ pub async fn binned_stream<SK>(
stream_kind: SK,
) -> Result<BinnedStreamRes<SK::TBinnedBins>, Error>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
if query.channel().backend != node_config.node.backend {
let err = Error::with_msg(format!(

15
disk/src/binnedstream.rs
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use crate::cache::pbvfs::PreBinnedScalarValueFetchedStream;
use crate::cache::{CacheUsage, PreBinnedQuery};
use crate::frame::makeframe::FrameType;
@@ -14,21 +14,18 @@ use std::task::{Context, Poll};
pub struct BinnedScalarStreamFromPreBinnedPatches<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
inp: Pin<
Box<
dyn Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>>
+ Send,
>,
Box<dyn Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, Error>> + Send>,
>,
_stream_kind: SK,
}
impl<SK> BinnedScalarStreamFromPreBinnedPatches<SK>
where
SK: BinnedStreamKind,
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>: FrameType,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, Error>: FrameType,
{
pub fn new(
patch_it: PreBinnedPatchIterator,
@@ -120,7 +117,7 @@ where
// Can I remove the whole type or keep for static check?
impl<SK> Stream for BinnedScalarStreamFromPreBinnedPatches<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Item = Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, Error>;

20
disk/src/cache.rs
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use crate::cache::pbv::PreBinnedValueByteStream;
use crate::frame::makeframe::FrameType;
use crate::merge::MergedMinMaxAvgScalarStream;
@@ -309,7 +309,7 @@ pub fn pre_binned_bytes_for_http<SK>(
stream_kind: SK,
) -> Result<PreBinnedValueByteStream<SK>, Error>
where
SK: BinnedStreamKind,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, err::Error>: FrameType,
{
if query.channel.backend != node_config.node.backend {
@@ -396,18 +396,18 @@ type T002<T> = Pin<Box<dyn Future<Output = Result<T001<T>, Error>> + Send>>;
pub struct MergedFromRemotes<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
tcp_establish_futs: Vec<T002<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>>,
nodein: Vec<Option<T001<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>>>,
merged: Option<T001<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>>,
tcp_establish_futs: Vec<T002<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>>,
nodein: Vec<Option<T001<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>>>,
merged: Option<T001<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>>,
completed: bool,
errored: bool,
}
impl<SK> MergedFromRemotes<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
pub fn new(evq: EventsQuery, perf_opts: PerfOpts, cluster: Cluster, stream_kind: SK) -> Self {
let mut tcp_establish_futs = vec![];
@@ -418,7 +418,7 @@ where
node.clone(),
stream_kind.clone(),
);
let f: T002<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>> = Box::pin(f);
let f: T002<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>> = Box::pin(f);
tcp_establish_futs.push(f);
}
let n = tcp_establish_futs.len();
@@ -434,9 +434,9 @@ where
impl<SK> Stream for MergedFromRemotes<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;

54
disk/src/cache/pbv.rs vendored
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::{Appendable, StreamItem};
use crate::binned::{BinnedStreamKind, RangeCompletableItem, WithLen};
use crate::binned::{RangeCompletableItem, StreamKind, WithLen};
use crate::cache::pbvfs::PreBinnedScalarValueFetchedStream;
use crate::cache::{CacheFileDesc, MergedFromRemotes, PreBinnedQuery, WrittenPbCache};
use crate::frame::makeframe::{make_frame, FrameType};
@@ -22,7 +22,7 @@ pub type PreBinnedValueByteStream<BK> = SCC<PreBinnedValueByteStreamInner<BK>>;
pub struct PreBinnedValueByteStreamInner<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
inp: PreBinnedValueStream<SK>,
}
@@ -33,8 +33,8 @@ pub fn pre_binned_value_byte_stream_new<SK>(
stream_kind: SK,
) -> PreBinnedValueByteStream<SK>
where
SK: BinnedStreamKind,
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>: FrameType,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>: FrameType,
{
let s1 = PreBinnedValueStream::new(query.clone(), node_config, stream_kind);
let s2 = PreBinnedValueByteStreamInner { inp: s1 };
@@ -43,8 +43,8 @@ where
impl<SK> Stream for PreBinnedValueByteStreamInner<SK>
where
SK: BinnedStreamKind,
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>: FrameType,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>: FrameType,
{
type Item = Result<Bytes, Error>;
@@ -52,10 +52,9 @@ where
use Poll::*;
match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => {
match make_frame::<
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>,
>(&item)
{
match make_frame::<Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>>(
&item,
) {
Ok(buf) => Ready(Some(Ok(buf.freeze()))),
Err(e) => Ready(Some(Err(e.into()))),
}
@@ -68,7 +67,7 @@ where
pub struct PreBinnedValueStream<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
query: PreBinnedQuery,
node_config: NodeConfigCached,
@@ -76,12 +75,8 @@ where
fut2: Option<
Pin<
Box<
dyn Stream<
Item = Result<
StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>,
err::Error,
>,
> + Send,
dyn Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>>
+ Send,
>,
>,
>,
@@ -93,16 +88,13 @@ where
errored: bool,
completed: bool,
streamlog: Streamlog,
values: <SK as BinnedStreamKind>::TBinnedBins,
values: <SK as StreamKind>::TBinnedBins,
write_fut: Option<Pin<Box<dyn Future<Output = Result<WrittenPbCache, Error>> + Send>>>,
read_cache_fut: Option<
Pin<
Box<
dyn Future<
Output = Result<
StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>,
err::Error,
>,
Output = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>,
> + Send,
>,
>,
@@ -112,8 +104,8 @@ where
impl<SK> PreBinnedValueStream<SK>
where
SK: BinnedStreamKind,
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>: FrameType,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>: FrameType,
{
pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: SK) -> Self {
Self {
@@ -129,7 +121,7 @@ where
errored: false,
completed: false,
streamlog: Streamlog::new(node_config.ix as u32),
values: <<SK as BinnedStreamKind>::TBinnedBins as Appendable>::empty(),
values: <<SK as StreamKind>::TBinnedBins as Appendable>::empty(),
write_fut: None,
read_cache_fut: None,
stream_kind,
@@ -164,7 +156,7 @@ where
self.node_config.node_config.cluster.clone(),
self.stream_kind.clone(),
);
let s1 = <SK as BinnedStreamKind>::xbinned_to_tbinned(s1, range);
let s1 = <SK as StreamKind>::xbinned_to_tbinned(s1, range);
self.fut2 = Some(Box::pin(s1));
}
@@ -239,10 +231,10 @@ where
impl<SK> Stream for PreBinnedValueStream<SK>
where
SK: BinnedStreamKind + Unpin,
Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>: FrameType,
SK: StreamKind + Unpin,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>: FrameType,
{
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, err::Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::TBinnedBins>>, err::Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -318,7 +310,7 @@ where
self.streamlog.append(Level::INFO, msg);
let values = std::mem::replace(
&mut self.values,
<<SK as BinnedStreamKind>::TBinnedBins as Appendable>::empty(),
<<SK as StreamKind>::TBinnedBins as Appendable>::empty(),
);
let fut = super::write_pb_cache_min_max_avg_scalar(
values,
@@ -371,7 +363,7 @@ where
match item {
Ok(file) => {
self.read_from_cache = true;
let fut = <<SK as BinnedStreamKind>::TBinnedBins as crate::binned::ReadableFromFile>::read_from_file(file)?;
let fut = <<SK as StreamKind>::TBinnedBins as crate::binned::ReadableFromFile>::read_from_file(file)?;
self.read_cache_fut = Some(Box::pin(fut));
continue 'outer;
}

8
disk/src/cache/pbvfs.rs vendored
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead, PreBinnedQuery};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::{decode_frame, FrameType};
@@ -14,7 +14,7 @@ use std::task::{Context, Poll};
pub struct PreBinnedScalarValueFetchedStream<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
uri: http::Uri,
resfut: Option<hyper::client::ResponseFuture>,
@@ -26,7 +26,7 @@ where
impl<SK> PreBinnedScalarValueFetchedStream<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached, stream_kind: &SK) -> Result<Self, Error> {
let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
@@ -53,7 +53,7 @@ where
// TODO change name, is now generic:
impl<SK> Stream for PreBinnedScalarValueFetchedStream<SK>
where
SK: BinnedStreamKind,
SK: StreamKind,
Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, err::Error>: FrameType,
{
type Item = Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, Error>;

24
disk/src/merge.rs
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::{Appendable, StatsItem, StreamItem};
use crate::binned::{BinnedStreamKind, PushableIndex, RangeCompletableItem, WithLen, WithTimestamps};
use crate::binned::{PushableIndex, RangeCompletableItem, StreamKind, WithLen, WithTimestamps};
use crate::streamlog::LogItem;
use err::Error;
use futures_core::Stream;
@@ -13,14 +13,14 @@ use std::task::{Context, Poll};
pub struct MergedMinMaxAvgScalarStream<S, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<SK::XBinnedEvents>>, Error>> + Unpin,
SK: BinnedStreamKind,
SK: StreamKind,
{
inps: Vec<S>,
current: Vec<MergedCurVal<<SK as BinnedStreamKind>::XBinnedEvents>>,
current: Vec<MergedCurVal<<SK as StreamKind>::XBinnedEvents>>,
ixs: Vec<usize>,
errored: bool,
completed: bool,
batch: <SK as BinnedStreamKind>::XBinnedEvents,
batch: <SK as StreamKind>::XBinnedEvents,
ts_last_emit: u64,
range_complete_observed: Vec<bool>,
range_complete_observed_all: bool,
@@ -34,7 +34,7 @@ where
impl<S, SK> MergedMinMaxAvgScalarStream<S, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<SK::XBinnedEvents>>, Error>> + Unpin,
SK: BinnedStreamKind,
SK: StreamKind,
{
pub fn new(inps: Vec<S>) -> Self {
let n = inps.len();
@@ -45,7 +45,7 @@ where
ixs: vec![0; n],
errored: false,
completed: false,
batch: <<SK as BinnedStreamKind>::XBinnedEvents as Appendable>::empty(),
batch: <<SK as StreamKind>::XBinnedEvents as Appendable>::empty(),
ts_last_emit: 0,
range_complete_observed: vec![false; n],
range_complete_observed_all: false,
@@ -125,10 +125,10 @@ where
// TODO change name, it is generic now:
impl<S, SK> Stream for MergedMinMaxAvgScalarStream<S, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>> + Unpin,
SK: StreamKind,
{
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -181,7 +181,7 @@ where
if self.batch.len() != 0 {
//let k = std::mem::replace(&mut self.batch, MinMaxAvgScalarEventBatch::empty());
//let ret = MinMaxAvgScalarEventBatchStreamItem::Values(k);
let emp = <<SK as BinnedStreamKind>::XBinnedEvents as Appendable>::empty();
let emp = <<SK as StreamKind>::XBinnedEvents as Appendable>::empty();
let ret = std::mem::replace(&mut self.batch, emp);
self.data_emit_complete = true;
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(ret)))))
@@ -191,7 +191,7 @@ where
}
} else {
assert!(lowest_ts >= self.ts_last_emit);
let emp = <<SK as BinnedStreamKind>::XBinnedEvents as Appendable>::empty();
let emp = <<SK as StreamKind>::XBinnedEvents as Appendable>::empty();
let mut local_batch = std::mem::replace(&mut self.batch, emp);
self.ts_last_emit = lowest_ts;
let rix = self.ixs[lowest_ix];
@@ -225,7 +225,7 @@ where
if self.batch.len() >= self.batch_size {
//let k = std::mem::replace(&mut self.batch, MinMaxAvgScalarEventBatch::empty());
//let ret = MinMaxAvgScalarEventBatchStreamItem::Values(k);
let emp = <<SK as BinnedStreamKind>::XBinnedEvents as Appendable>::empty();
let emp = <<SK as StreamKind>::XBinnedEvents as Appendable>::empty();
let ret = std::mem::replace(&mut self.batch, emp);
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(ret)))))
} else {

6
disk/src/raw.rs
View File

@@ -6,7 +6,7 @@ to request such data from nodes.
*/
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::binned::{RangeCompletableItem, StreamKind};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::{make_frame, make_term_frame};
use crate::raw::bffr::EventsFromFrames;
@@ -44,14 +44,14 @@ pub async fn x_processed_stream_from_node<SK>(
) -> Result<
Pin<
Box<
dyn Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>>
dyn Stream<Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>>
+ Send,
>,
>,
Error,
>
where
SK: BinnedStreamKind,
SK: StreamKind,
{
let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
let qjs = serde_json::to_string(&query)?;

17
disk/src/raw/bffr.rs
View File

@@ -1,5 +1,5 @@
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem, XBinnedEvents};
use crate::binned::{RangeCompletableItem, StreamKind, XBinnedEvents};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::decode_frame;
use err::Error;
@@ -13,7 +13,7 @@ use tokio::io::AsyncRead;
pub struct EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
SK: StreamKind,
{
inp: InMemoryFrameAsyncReadStream<T>,
errored: bool,
@@ -24,7 +24,7 @@ where
impl<T, SK> EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
SK: StreamKind,
{
pub fn new(inp: InMemoryFrameAsyncReadStream<T>, stream_kind: SK) -> Self {
Self {
@@ -39,11 +39,11 @@ where
impl<T, SK> Stream for EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
SK: StreamKind,
// TODO see binned.rs better to express it on trait?
//Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>: FrameType,
{
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -60,13 +60,10 @@ where
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(frame) => {
match decode_frame::<
Result<
StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>,
Error,
>,
Result<StreamItem<RangeCompletableItem<<SK as StreamKind>::XBinnedEvents>>, Error>,
>(
&frame,
<<SK as BinnedStreamKind>::XBinnedEvents as XBinnedEvents<SK>>::frame_type(),
<<SK as StreamKind>::XBinnedEvents as XBinnedEvents<SK>>::frame_type(),
) {
Ok(item) => match item {
Ok(item) => Ready(Some(Ok(item))),

9
disk/src/raw/conn.rs
View File

@@ -2,7 +2,7 @@ use crate::agg::binnedx::IntoBinnedXBins1;
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::streams::StreamItem;
use crate::agg::IntoDim1F32Stream;
use crate::binned::{BinnedStreamKind, BinnedStreamKindScalar, RangeCompletableItem};
use crate::binned::{BinnedStreamKindScalar, RangeCompletableItem, StreamKind};
use crate::eventblobs::EventBlobsComplete;
use crate::eventchunker::EventChunkerConf;
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
@@ -189,7 +189,7 @@ async fn events_conn_handler_inner_try(
AggKind::DimXBins1 => {
match make_frame::<
Result<
StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as BinnedStreamKind>::XBinnedEvents>>,
StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as StreamKind>::XBinnedEvents>>,
Error,
>,
>(&item)
@@ -205,10 +205,7 @@ async fn events_conn_handler_inner_try(
}
// TODO define this case:
AggKind::DimXBinsN(_xbincount) => match make_frame::<
Result<
StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as BinnedStreamKind>::XBinnedEvents>>,
Error,
>,
Result<StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as StreamKind>::XBinnedEvents>>, Error>,
>(err::todoval())
{
Ok(buf) => match netout.write_all(&buf).await {