daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor number pipeline for simple cases

Browse Source
This commit is contained in:
Dominik Werder
2021-06-08 15:36:54 +02:00
parent e281a5c8fd
commit 7b40938427
17 changed files with 920 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
disk/src/agg/binnedt4.rs
View File

@@ -1,22 +1,25 @@
use crate::agg::enp::XBinnedScalarEvents;
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::StreamItem;
use crate::agg::streams::{Appendable, StreamItem};
use crate::binned::{
BinsTimeBinner, EventsTimeBinner, EventsTimeBinnerAggregator, MinMaxAvgAggregator, MinMaxAvgBins, NumOps,
RangeCompletableItem, RangeOverlapInfo, SingleXBinAggregator,
BinsTimeBinner, EventsTimeBinner, EventsTimeBinnerAggregator, FilterFittingInside, MinMaxAvgAggregator,
MinMaxAvgBins, NumOps, RangeCompletableItem, RangeOverlapInfo, ReadPbv, ReadableFromFile, SingleXBinAggregator,
};
use crate::decode::EventValues;
use crate::decode::{EventValues, MinMaxAvgScalarEventBatchGen};
use crate::frame::makeframe::Framable;
use crate::Sitemty;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{BinnedRange, NanoRange};
use serde::Serialize;
use std::collections::VecDeque;
use std::marker::PhantomData;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::fs::File;
pub struct DefaultScalarEventsTimeBinner<VT> {
_m1: PhantomData<VT>,
@@ -141,17 +144,33 @@ impl Agg3 {
}
}
pub struct TBinnerStream<S, ETB>
where
S: Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Input>> + Send + Unpin + 'static,
ETB: EventsTimeBinner + Send + Unpin + 'static,
pub trait TimeBinnableTypeAggregator: Send {
type Input: TimeBinnableType;
type Output: TimeBinnableType;
fn range(&self) -> &NanoRange;
fn ingest(&mut self, item: &Self::Input);
fn result(self) -> Self::Output;
}
pub trait TimeBinnableType:
Send + Unpin + RangeOverlapInfo + FilterFittingInside + Appendable + Serialize + ReadableFromFile
{
inp: S,
type Output: TimeBinnableType;
type Aggregator: TimeBinnableTypeAggregator<Input = Self, Output = Self::Output> + Send + Unpin;
fn aggregator(range: NanoRange) -> Self::Aggregator;
}
pub struct TBinnerStream<S, TBT>
where
S: Stream<Item = Sitemty<TBT>>,
TBT: TimeBinnableType,
{
inp: Pin<Box<S>>,
spec: BinnedRange,
curbin: u32,
left: Option<Poll<Option<Sitemty<<ETB as EventsTimeBinner>::Input>>>>,
aggtor: Option<<ETB as EventsTimeBinner>::Aggregator>,
tmp_agg_results: VecDeque<<<ETB as EventsTimeBinner>::Aggregator as EventsTimeBinnerAggregator>::Output>,
left: Option<Poll<Option<Sitemty<TBT>>>>,
aggtor: Option<<TBT as TimeBinnableType>::Aggregator>,
tmp_agg_results: VecDeque<<<TBT as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output>,
inp_completed: bool,
all_bins_emitted: bool,
range_complete_observed: bool,
@@ -160,19 +179,19 @@ where
completed: bool,
}
impl<S, ETB> TBinnerStream<S, ETB>
impl<S, TBT> TBinnerStream<S, TBT>
where
S: Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Input>> + Send + Unpin + 'static,
ETB: EventsTimeBinner,
S: Stream<Item = Sitemty<TBT>> + Send + Unpin + 'static,
TBT: TimeBinnableType,
{
pub fn new(inp: S, spec: BinnedRange) -> Self {
let range = spec.get_range(0);
Self {
inp,
inp: Box::pin(inp),
spec,
curbin: 0,
left: None,
aggtor: Some(<ETB as EventsTimeBinner>::aggregator(range)),
aggtor: Some(<TBT as TimeBinnableType>::aggregator(range)),
tmp_agg_results: VecDeque::new(),
inp_completed: false,
all_bins_emitted: false,
@@ -183,7 +202,7 @@ where
}
}
fn cur(&mut self, cx: &mut Context) -> Poll<Option<Sitemty<<ETB as EventsTimeBinner>::Input>>> {
fn cur(&mut self, cx: &mut Context) -> Poll<Option<Sitemty<TBT>>> {
if let Some(cur) = self.left.take() {
cur
} else if self.inp_completed {
@@ -194,13 +213,13 @@ where
}
}
// TODO handle unwrap error, or use a mem replace type instead of option:
fn cycle_current_bin(&mut self) {
self.curbin += 1;
let range = self.spec.get_range(self.curbin);
let ret = self
.aggtor
.replace(<ETB as EventsTimeBinner>::aggregator(range))
// TODO handle None case, or remove Option if Agg is always present
.replace(<TBT as TimeBinnableType>::aggregator(range))
.unwrap()
.result();
// TODO should we accumulate bins before emit? Maybe not, we want to stay responsive.
@@ -213,8 +232,9 @@ where
fn handle(
&mut self,
cur: Poll<Option<Sitemty<<ETB as EventsTimeBinner>::Input>>>,
) -> Option<Poll<Option<Sitemty<<ETB as EventsTimeBinner>::Output>>>> {
cur: Poll<Option<Sitemty<TBT>>>,
) -> Option<Poll<Option<Sitemty<<<TBT as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output>>>>
{
use Poll::*;
match cur {
Ready(Some(Ok(item))) => match item {
@@ -228,7 +248,7 @@ where
RangeCompletableItem::Data(item) => {
if self.all_bins_emitted {
// Just drop the item because we will not emit anymore data.
// Could also at least gather some stats.
// TODO gather stats.
None
} else {
let ag = self.aggtor.as_mut().unwrap();
@@ -273,18 +293,20 @@ where
}
}
impl<S, ETB> Stream for TBinnerStream<S, ETB>
impl<S, TBT> Stream for TBinnerStream<S, TBT>
where
S: Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Input>> + Send + Unpin + 'static,
ETB: EventsTimeBinner + Send + Unpin + 'static,
S: Stream<Item = Sitemty<TBT>> + Send + Unpin + 'static,
TBT: TimeBinnableType + Send + Unpin + 'static,
<TBT as TimeBinnableType>::Aggregator: Unpin,
<<TBT as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output: Unpin,
{
type Item = Sitemty<<ETB as EventsTimeBinner>::Output>;
type Item = Sitemty<<<TBT as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
'outer: loop {
break if self.completed {
panic!("IntoBinnedTDefaultStream poll_next on completed");
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
Ready(None)
@@ -310,3 +332,57 @@ where
}
}
}
pub struct MinMaxAvgScalarEventBatchGenAggregator<NTY>
where
NTY: NumOps,
{
_m1: PhantomData<NTY>,
}
impl<NTY> TimeBinnableTypeAggregator for MinMaxAvgScalarEventBatchGenAggregator<NTY>
where
NTY: NumOps,
{
type Input = MinMaxAvgScalarEventBatchGen<NTY>;
type Output = MinMaxAvgScalarEventBatchGen<NTY>;
fn range(&self) -> &NanoRange {
todo!()
}
fn ingest(&mut self, item: &Self::Input) {
todo!()
}
fn result(self) -> Self::Output {
todo!()
}
}
impl<NTY> ReadableFromFile for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn read_from_file(file: File) -> Result<ReadPbv<Self>, Error> {
todo!()
}
fn from_buf(buf: &[u8]) -> Result<Self, Error> {
todo!()
}
}
// TODO this is just dummy, do I use this in the refactored code?
impl<NTY> TimeBinnableType for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
// TODO Output is just dummy, because this type is probably unused anyways.
type Output = MinMaxAvgScalarEventBatchGen<NTY>;
type Aggregator = MinMaxAvgScalarEventBatchGenAggregator<NTY>;
fn aggregator(range: NanoRange) -> Self::Aggregator {
todo!()
}
}

134
disk/src/agg/enp.rs
View File

@@ -1,10 +1,16 @@
use crate::agg::binnedt4::{TimeBinnableType, TimeBinnableTypeAggregator};
use crate::agg::streams::Appendable;
use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, RangeOverlapInfo, WithLen, WithTimestamps};
use crate::decode::EventValues;
use crate::binned::{
EventsNodeProcessor, FilterFittingInside, MinMaxAvgBins, NumOps, PushableIndex, RangeOverlapInfo, ReadPbv,
ReadableFromFile, WithLen, WithTimestamps,
};
use crate::decode::{EventValues, MinMaxAvgScalarEventBatchGen};
use err::Error;
use netpod::NanoRange;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;
use tokio::fs::File;
pub struct Identity<NTY> {
_m1: PhantomData<NTY>,
@@ -78,6 +84,12 @@ impl<NTY> RangeOverlapInfo for XBinnedScalarEvents<NTY> {
}
}
impl<NTY> FilterFittingInside for XBinnedScalarEvents<NTY> {
fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
todo!()
}
}
impl<NTY> PushableIndex for XBinnedScalarEvents<NTY>
where
NTY: NumOps,
@@ -108,6 +120,124 @@ where
}
}
impl<NTY> ReadableFromFile for XBinnedScalarEvents<NTY>
where
NTY: NumOps,
{
fn read_from_file(file: File) -> Result<ReadPbv<Self>, Error> {
todo!()
}
fn from_buf(buf: &[u8]) -> Result<Self, Error> {
todo!()
}
}
impl<NTY> TimeBinnableType for XBinnedScalarEvents<NTY>
where
NTY: NumOps,
{
type Output = MinMaxAvgBins<NTY>;
type Aggregator = XBinnedScalarEventsAggregator<NTY>;
fn aggregator(range: NanoRange) -> Self::Aggregator {
Self::Aggregator::new(range)
}
}
pub struct XBinnedScalarEventsAggregator<NTY>
where
NTY: NumOps,
{
range: NanoRange,
min: Option<NTY>,
max: Option<NTY>,
sumc: u32,
sum: f32,
}
impl<NTY> XBinnedScalarEventsAggregator<NTY>
where
NTY: NumOps,
{
pub fn new(range: NanoRange) -> Self {
Self {
range,
min: None,
max: None,
sumc: 0,
sum: 0f32,
}
}
}
impl<NTY> TimeBinnableTypeAggregator for XBinnedScalarEventsAggregator<NTY>
where
NTY: NumOps,
{
type Input = XBinnedScalarEvents<NTY>;
type Output = MinMaxAvgBins<NTY>;
fn range(&self) -> &NanoRange {
&self.range
}
fn ingest(&mut self, item: &Self::Input) {
for i1 in 0..item.tss.len() {
let ts = item.tss[i1];
if ts < self.range.beg {
continue;
} else if ts >= self.range.end {
continue;
} else {
self.min = match self.min {
None => Some(item.mins[i1]),
Some(min) => {
if item.mins[i1] < min {
Some(item.mins[i1])
} else {
Some(min)
}
}
};
self.max = match self.max {
None => Some(item.maxs[i1]),
Some(max) => {
if item.maxs[i1] > max {
Some(item.maxs[i1])
} else {
Some(max)
}
}
};
let x = item.avgs[i1];
if x.is_nan() {
} else {
self.sum += x;
self.sumc += 1;
}
}
}
}
fn result(self) -> Self::Output {
let avg = if self.sumc == 0 {
None
} else {
Some(self.sum / self.sumc as f32)
};
Self::Output {
ts1s: vec![self.range.beg],
ts2s: vec![self.range.end],
// TODO
counts: vec![0],
mins: vec![self.min],
maxs: vec![self.max],
avgs: vec![avg],
}
}
}
pub struct WaveXBinner<NTY> {
_m1: PhantomData<NTY>,
}

304
disk/src/binned.rs
View File

@@ -1,12 +1,16 @@
use crate::agg::binnedt::AggregatableTdim;
use crate::agg::binnedt2::AggregatableTdim2;
use crate::agg::binnedt3::{Agg3, BinnedT3Stream};
use crate::agg::binnedt4::{DefaultScalarEventsTimeBinner, DefaultSingleXBinTimeBinner};
use crate::agg::binnedt4::{
DefaultScalarEventsTimeBinner, DefaultSingleXBinTimeBinner, TBinnerStream, TimeBinnableType,
TimeBinnableTypeAggregator,
};
use crate::agg::enp::{Identity, WaveXBinner, XBinnedScalarEvents};
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::{Appendable, Collectable, Collected, StreamItem, ToJsonResult};
use crate::agg::{Fits, FitsInside};
use crate::binned::binnedfrompbv::BinnedFromPreBinned;
use crate::binned::query::{BinnedQuery, PreBinnedQuery};
use crate::binned::scalar::binned_stream;
use crate::binnedstream::{BinnedScalarStreamFromPreBinnedPatches, BoxedStream};
@@ -16,15 +20,16 @@ use crate::decode::{
LittleEndian, NumFromBytes,
};
use crate::frame::makeframe::{Framable, FrameType, SubFrId};
use crate::merge::mergefromremote::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::raw::EventsQuery;
use crate::Sitemty;
use bytes::Bytes;
use chrono::{TimeZone, Utc};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use futures_util::{FutureExt, StreamExt};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{
AggKind, BinnedRange, ByteOrder, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator,
PreBinnedPatchRange, ScalarType, Shape,
@@ -42,6 +47,7 @@ use std::time::Duration;
use tokio::fs::File;
use tokio::io::{AsyncRead, ReadBuf};
pub mod binnedfrompbv;
pub mod pbv;
pub mod pbv2;
pub mod prebinned;
@@ -179,10 +185,19 @@ where
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Output> + 'static,
ETB: EventsTimeBinner<Input = <ENP as EventsNodeProcessor>::Output> + 'static,
<ENP as EventsNodeProcessor>::Output: PushableIndex + Appendable + 'static,
<ENP as EventsNodeProcessor>::Output: TimeBinnableType + PushableIndex + Appendable + 'static,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output:
TimeBinnableType<Output = <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output> + Unpin,
<ETB as EventsTimeBinner>::Output: Serialize + ReadableFromFile + 'static,
Sitemty<
<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output,
>: Framable,
// TODO require these things in general?
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<ETB as EventsTimeBinner>::Output>: Framable,
Sitemty<<ETB as EventsTimeBinner>::Output>: FrameType + Framable + DeserializeOwned,
Sitemty<
<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Aggregator as TimeBinnableTypeAggregator>::Output,
>: FrameType + Framable + DeserializeOwned,
{
// TODO construct the binned pipeline:
// Either take from prebinned sub sstream, or directly from a merged.
@@ -212,11 +227,7 @@ where
);
return Err(Error::with_msg(msg));
}
// TODO
// Must generify the BinnedScalarStreamFromPreBinnedPatches.
// Copy code and introduce type parameters.
let s = BinnedScalarStreamFromPreBinnedPatches::new(
let s = BinnedFromPreBinned::<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>::new(
PreBinnedPatchIterator::from_range(pre_range),
query.channel().clone(),
range.clone(),
@@ -225,15 +236,9 @@ where
node_config,
query.disk_stats_every().clone(),
query.report_error(),
self.clone(),
)?;
let s = BoxedStream::new(Box::pin(s))?;
let ret = BinnedStreamRes {
binned_stream: s,
range,
};
Ok(ret)
)?
.map(|item| Box::new(item) as Box<dyn Framable>);
Ok(Box::pin(s))
}
Ok(None) => {
info!(
@@ -245,24 +250,13 @@ where
range: query.range().clone(),
agg_kind: query.agg_kind().clone(),
};
// TODO do I need to set up more transformations or binning to deliver the requested data?
//let s = SK::new_binned_from_merged(&stream_kind, evq, perf_opts, range.clone(), node_config)?;
// TODO adapt the usage the same way how I do in prebinned.rs:
let s = MergedFromRemotes2::new(evq, perf_opts, node_config.node_config.cluster.clone(), self.clone());
let s = Self::xbinned_to_tbinned(s, range);
let s = BoxedStream::new(Box::pin(s))?;
let ret = BinnedStreamRes {
binned_stream: s,
range,
};
Ok(ret)
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
let s = StreamExt::map(s, |item| Box::new(item) as Box<dyn Framable>);
Ok(Box::pin(s))
}
Err(e) => Err(e),
}
err::todoval()
}
fn make_num_pipeline_nty_end<NTY, END>(
@@ -719,11 +713,21 @@ impl TBinnedBins for MinMaxAvgScalarBinBatch {
}
pub trait NumOps:
Sized + Copy + Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + DeserializeOwned
Sized + Copy + Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
{
}
impl<T> NumOps for T where
T: Sized + Copy + Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + DeserializeOwned
T: Sized
+ Copy
+ Send
+ Unpin
+ Zero
+ AsPrimitive<f32>
+ Bounded
+ PartialOrd
+ SubFrId
+ Serialize
+ DeserializeOwned
{
}
@@ -735,15 +739,16 @@ pub trait EventsDecoder {
pub trait EventsNodeProcessor: Send + Unpin {
type Input;
type Output: Send + Unpin + DeserializeOwned + WithTimestamps;
type Output: Send + Unpin + DeserializeOwned + WithTimestamps + TimeBinnableType;
fn process(inp: EventValues<Self::Input>) -> Self::Output;
}
pub trait TimeBins: Send + Unpin + WithLen + Appendable {
pub trait TimeBins: Send + Unpin + WithLen + Appendable + FilterFittingInside {
fn ts1s(&self) -> &Vec<u64>;
fn ts2s(&self) -> &Vec<u64>;
}
// TODO remove in favor of the one in binnedt4
pub trait EventsTimeBinner: Send + Unpin {
type Input: Unpin + RangeOverlapInfo;
type Output: TimeBins;
@@ -751,6 +756,7 @@ pub trait EventsTimeBinner: Send + Unpin {
fn aggregator(range: NanoRange) -> Self::Aggregator;
}
// TODO remove in favor of the one in binnedt4
pub trait EventsTimeBinnerAggregator: Send {
type Input: Unpin;
type Output: Unpin;
@@ -767,12 +773,31 @@ pub trait BinsTimeBinner {
#[derive(Serialize, Deserialize)]
pub struct MinMaxAvgBins<NTY> {
ts1s: Vec<u64>,
ts2s: Vec<u64>,
counts: Vec<u64>,
mins: Vec<Option<NTY>>,
maxs: Vec<Option<NTY>>,
avgs: Vec<Option<f32>>,
pub ts1s: Vec<u64>,
pub ts2s: Vec<u64>,
pub counts: Vec<u64>,
pub mins: Vec<Option<NTY>>,
pub maxs: Vec<Option<NTY>>,
pub avgs: Vec<Option<f32>>,
}
impl<NTY> std::fmt::Debug for MinMaxAvgBins<NTY>
where
NTY: std::fmt::Debug,
{
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
fmt,
"MinMaxAvgBins count {} ts1s {:?} ts2s {:?} counts {:?} mins {:?} maxs {:?} avgs {:?}",
self.ts1s.len(),
self.ts1s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.ts2s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.counts,
self.mins,
self.maxs,
self.avgs,
)
}
}
impl<NTY> MinMaxAvgBins<NTY> {
@@ -788,6 +813,62 @@ impl<NTY> MinMaxAvgBins<NTY> {
}
}
impl<NTY> FitsInside for MinMaxAvgBins<NTY> {
fn fits_inside(&self, range: NanoRange) -> Fits {
if self.ts1s.is_empty() {
Fits::Empty
} else {
let t1 = *self.ts1s.first().unwrap();
let t2 = *self.ts2s.last().unwrap();
if t2 <= range.beg {
Fits::Lower
} else if t1 >= range.end {
Fits::Greater
} else if t1 < range.beg && t2 > range.end {
Fits::PartlyLowerAndGreater
} else if t1 < range.beg {
Fits::PartlyLower
} else if t2 > range.end {
Fits::PartlyGreater
} else {
Fits::Inside
}
}
}
}
impl<NTY> FilterFittingInside for MinMaxAvgBins<NTY> {
fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
match self.fits_inside(fit_range) {
Fits::Inside | Fits::PartlyGreater | Fits::PartlyLower | Fits::PartlyLowerAndGreater => Some(self),
_ => None,
}
}
}
impl<NTY> RangeOverlapInfo for MinMaxAvgBins<NTY> {
fn ends_before(&self, range: NanoRange) -> bool {
match self.ts2s.last() {
Some(&ts) => ts <= range.beg,
None => true,
}
}
fn ends_after(&self, range: NanoRange) -> bool {
match self.ts2s.last() {
Some(&ts) => ts > range.end,
None => panic!(),
}
}
fn starts_after(&self, range: NanoRange) -> bool {
match self.ts1s.first() {
Some(&ts) => ts >= range.end,
None => panic!(),
}
}
}
impl<NTY> TimeBins for MinMaxAvgBins<NTY>
where
NTY: NumOps,
@@ -840,6 +921,18 @@ where
}
}
impl<NTY> TimeBinnableType for MinMaxAvgBins<NTY>
where
NTY: NumOps,
{
type Output = MinMaxAvgBins<NTY>;
type Aggregator = MinMaxAvgBinsAggregator<NTY>;
fn aggregator(range: NanoRange) -> Self::Aggregator {
Self::Aggregator::new(range)
}
}
pub struct MinMaxAvgAggregator<NTY> {
range: NanoRange,
count: u32,
@@ -860,6 +953,28 @@ impl<NTY> MinMaxAvgAggregator<NTY> {
}
}
// TODO rename to EventValuesAggregator
impl<NTY> TimeBinnableTypeAggregator for MinMaxAvgAggregator<NTY>
where
NTY: NumOps,
{
type Input = EventValues<NTY>;
type Output = MinMaxAvgBins<NTY>;
fn range(&self) -> &NanoRange {
&self.range
}
fn ingest(&mut self, item: &Self::Input) {
todo!()
}
fn result(self) -> Self::Output {
todo!()
}
}
// TODO after refactor get rid of this impl:
impl<NTY> EventsTimeBinnerAggregator for MinMaxAvgAggregator<NTY>
where
NTY: NumOps,
@@ -880,6 +995,107 @@ where
}
}
pub struct MinMaxAvgBinsAggregator<NTY> {
range: NanoRange,
count: u32,
min: Option<NTY>,
max: Option<NTY>,
avg: Option<f32>,
sum: f32,
sumc: u32,
}
impl<NTY> MinMaxAvgBinsAggregator<NTY> {
pub fn new(range: NanoRange) -> Self {
Self {
range,
// TODO: count events here?
count: 0,
min: None,
max: None,
avg: None,
sum: 0f32,
sumc: 0,
}
}
}
impl<NTY> TimeBinnableTypeAggregator for MinMaxAvgBinsAggregator<NTY>
where
NTY: NumOps,
{
type Input = MinMaxAvgBins<NTY>;
type Output = MinMaxAvgBins<NTY>;
fn range(&self) -> &NanoRange {
&self.range
}
fn ingest(&mut self, item: &Self::Input) {
for i1 in 0..item.ts1s.len() {
if item.ts2s[i1] <= self.range.beg {
continue;
} else if item.ts1s[i1] >= self.range.end {
continue;
} else {
self.min = match self.min {
None => item.mins[i1],
Some(min) => match item.mins[i1] {
None => Some(min),
Some(v) => {
if v < min {
Some(v)
} else {
Some(min)
}
}
},
};
self.max = match self.max {
None => item.maxs[i1],
Some(max) => match item.maxs[i1] {
None => Some(max),
Some(v) => {
if v > max {
Some(v)
} else {
Some(max)
}
}
},
};
match item.avgs[i1] {
None => {}
Some(v) => {
if v.is_nan() {
} else {
self.sum += v;
self.sumc += 1;
}
}
}
}
}
}
fn result(self) -> Self::Output {
let avg = if self.sumc == 0 {
None
} else {
Some(self.sum / self.sumc as f32)
};
Self::Output {
ts1s: vec![self.range.beg],
ts2s: vec![self.range.end],
// TODO
counts: vec![0],
mins: vec![self.min],
maxs: vec![self.max],
avgs: vec![avg],
}
}
}
pub struct SingleXBinAggregator<NTY> {
range: NanoRange,
count: u32,

261
disk/src/binned/binnedfrompbv.rs Normal file
View File

@@ -0,0 +1,261 @@
use crate::agg::binnedt4::{TBinnerStream, TimeBinnableType, TimeBinnableTypeAggregator};
use crate::agg::streams::StreamItem;
use crate::binned::query::{CacheUsage, PreBinnedQuery};
use crate::binned::{EventsTimeBinner, RangeCompletableItem};
use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::{decode_frame, FrameType};
use crate::Sitemty;
use err::Error;
use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use http::{StatusCode, Uri};
use netpod::log::*;
use netpod::{AggKind, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts, PreBinnedPatchIterator};
use serde::de::DeserializeOwned;
use serde::Deserialize;
use std::future::ready;
use std::marker::PhantomData;
use std::pin::Pin;
use std::task::{Context, Poll};
pub struct FetchedPreBinned<TBT> {
uri: Uri,
resfut: Option<hyper::client::ResponseFuture>,
res: Option<InMemoryFrameAsyncReadStream<HttpBodyAsAsyncRead>>,
errored: bool,
completed: bool,
_m1: PhantomData<TBT>,
}
impl<TBT> FetchedPreBinned<TBT> {
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
let nodeix = node_ix_for_patch(&query.patch(), &query.channel(), &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize];
let uri: hyper::Uri = format!(
"http://{}:{}/api/4/prebinned?{}",
node.host,
node.port,
query.make_query_string()
)
.parse()?;
let ret = Self {
uri,
resfut: None,
res: None,
errored: false,
completed: false,
_m1: PhantomData,
};
Ok(ret)
}
}
impl<TBT> Stream for FetchedPreBinned<TBT>
where
TBT: TimeBinnableType,
Sitemty<TBT>: FrameType + DeserializeOwned,
{
type Item = Sitemty<TBT>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
'outer: loop {
break if self.completed {
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
return Ready(None);
} else if let Some(res) = self.res.as_mut() {
match res.poll_next_unpin(cx) {
Ready(Some(Ok(item))) => match item {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(item) => match decode_frame::<Sitemty<TBT>>(&item) {
Ok(Ok(item)) => Ready(Some(Ok(item))),
Ok(Err(e)) => {
self.errored = true;
Ready(Some(Err(e)))
}
Err(e) => {
self.errored = true;
Ready(Some(Err(e)))
}
},
},
Ready(Some(Err(e))) => {
self.errored = true;
Ready(Some(Err(e)))
}
Ready(None) => {
self.completed = true;
Ready(None)
}
Pending => Pending,
}
} else if let Some(resfut) = self.resfut.as_mut() {
match resfut.poll_unpin(cx) {
Ready(res) => match res {
Ok(res) => {
if res.status() == StatusCode::OK {
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
self.res = Some(s2);
continue 'outer;
} else {
error!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
);
let e = Error::with_msg(format!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
));
self.errored = true;
Ready(Some(Err(e)))
}
}
Err(e) => {
error!("PreBinnedValueStream error in stream {:?}", e);
self.errored = true;
Ready(Some(Err(e.into())))
}
},
Pending => Pending,
}
} else {
match hyper::Request::builder()
.method(http::Method::GET)
.uri(&self.uri)
.body(hyper::Body::empty())
{
Ok(req) => {
let client = hyper::Client::new();
self.resfut = Some(client.request(req));
continue 'outer;
}
Err(e) => {
self.errored = true;
Ready(Some(Err(e.into())))
}
}
};
}
}
}
/// Generate bins from a range of pre-binned patches.
///
/// Takes an iterator over the necessary patches.
pub struct BinnedFromPreBinned<TBT>
where
TBT: TimeBinnableType,
{
// TODO get rid of box:
inp: Pin<Box<dyn Stream<Item = Sitemty<TBT>> + Send>>,
_m1: PhantomData<TBT>,
}
impl<TBT> BinnedFromPreBinned<TBT>
where
TBT: TimeBinnableType<Output = TBT> + Unpin + 'static,
Sitemty<TBT>: FrameType + DeserializeOwned,
{
pub fn new(
patch_it: PreBinnedPatchIterator,
channel: Channel,
range: BinnedRange,
agg_kind: AggKind,
cache_usage: CacheUsage,
node_config: &NodeConfigCached,
disk_stats_every: ByteSize,
report_error: bool,
) -> Result<Self, Error> {
let patches: Vec<_> = patch_it.collect();
let mut sp = String::new();
if false {
// Convert this to a StreamLog message:
for (i, p) in patches.iter().enumerate() {
use std::fmt::Write;
write!(sp, " • patch {:2} {:?}\n", i, p)?;
}
info!("Using these pre-binned patches:\n{}", sp);
}
let pmax = patches.len();
let inp = futures_util::stream::iter(patches.into_iter().enumerate())
.map({
let node_config = node_config.clone();
move |(pix, patch)| {
let query = PreBinnedQuery::new(
patch,
channel.clone(),
agg_kind.clone(),
cache_usage.clone(),
disk_stats_every.clone(),
report_error,
);
let ret: Pin<Box<dyn Stream<Item = _> + Send>> =
match FetchedPreBinned::<TBT>::new(&query, &node_config) {
Ok(stream) => Box::pin(stream.map(move |q| (pix, q))),
Err(e) => {
error!("error from PreBinnedValueFetchedStream::new {:?}", e);
Box::pin(futures_util::stream::iter(vec![(pix, Err(e))]))
}
};
ret
}
})
.flatten()
.filter_map({
let range = range.clone();
move |(pix, k)| {
let fit_range = range.full_range();
let g = match k {
Ok(item) => match item {
StreamItem::Log(item) => Some(Ok(StreamItem::Log(item))),
StreamItem::Stats(item) => Some(Ok(StreamItem::Stats(item))),
StreamItem::DataItem(item) => match item {
RangeCompletableItem::RangeComplete => {
if pix + 1 == pmax {
Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)))
} else {
None
}
}
RangeCompletableItem::Data(item) => {
match crate::binned::FilterFittingInside::filter_fitting_inside(item, fit_range) {
Some(item) => Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))),
None => None,
}
}
},
},
Err(e) => Some(Err(e)),
};
ready(g)
}
});
// TODO TBinnerStream is for T-binning events.
// But here, we need to bin bins into bigger bins.
// The logic in TBinnerStream is actually the same I think..
// Reuse??
let inp = TBinnerStream::<_, TBT>::new(inp, range);
Ok(Self {
inp: Box::pin(inp),
_m1: PhantomData,
})
}
}
impl<TBT> Stream for BinnedFromPreBinned<TBT>
where
TBT: TimeBinnableType,
{
type Item = Sitemty<TBT>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.inp.poll_next_unpin(cx)
}
}

51
disk/src/binned/pbv.rs
View File

@@ -1,4 +1,4 @@
use crate::agg::binnedt4::TBinnerStream;
use crate::agg::binnedt4::{TBinnerStream, TimeBinnableType};
use crate::agg::streams::{Appendable, StreamItem};
use crate::binned::query::{CacheUsage, PreBinnedQuery};
use crate::binned::{
@@ -9,7 +9,7 @@ use crate::cache::pbvfs::PreBinnedScalarValueFetchedStream;
use crate::cache::{write_pb_cache_min_max_avg_scalar, CacheFileDesc, MergedFromRemotes, WrittenPbCache};
use crate::decode::{Endianness, EventValueFromBytes, EventValueShape, NumFromBytes};
use crate::frame::makeframe::{make_frame, FrameType};
use crate::merge::mergefromremote::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::raw::EventsQuery;
use crate::streamlog::Streamlog;
use crate::Sitemty;
@@ -40,7 +40,9 @@ where
query: PreBinnedQuery,
node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<File, io::Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Output>> + Send>>>,
fut2: Option<
Pin<Box<dyn Stream<Item = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>> + Send>>,
>,
read_from_cache: bool,
cache_written: bool,
data_complete: bool,
@@ -49,9 +51,15 @@ where
errored: bool,
completed: bool,
streamlog: Streamlog,
values: <ETB as EventsTimeBinner>::Output,
values: <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output,
write_fut: Option<Pin<Box<dyn Future<Output = Result<WrittenPbCache, Error>> + Send>>>,
read_cache_fut: Option<Pin<Box<dyn Future<Output = Sitemty<<ETB as EventsTimeBinner>::Output>> + Send>>>,
read_cache_fut: Option<
Pin<
Box<
dyn Future<Output = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>> + Send,
>,
>,
>,
_m1: PhantomData<NTY>,
_m2: PhantomData<END>,
_m3: PhantomData<EVS>,
@@ -84,7 +92,8 @@ where
errored: false,
completed: false,
streamlog: Streamlog::new(node_config.ix as u32),
values: <<ETB as EventsTimeBinner>::Output as Appendable>::empty(),
// TODO use alias via some trait associated type:
values: <<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output as Appendable>::empty(),
write_fut: None,
read_cache_fut: None,
_m1: PhantomData,
@@ -97,7 +106,10 @@ where
fn setup_merged_from_remotes(
&mut self,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Output>> + Send>>, Error> {
) -> Result<
Pin<Box<dyn Stream<Item = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>> + Send>>,
Error,
> {
let evq = EventsQuery {
channel: self.query.channel().clone(),
range: self.query.patch().patch_range(),
@@ -117,24 +129,18 @@ where
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
// TODO copy the MergedFromRemotes and adapt...
let s1 = MergedFromRemotes2::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
// TODO
// Go from ENP values to a T-binned stream...
// Most of the algo is static same.
// What varies: init aggregator for next T-bin.
let ret = TBinnerStream::<_, ETB>::new(s1, range);
//let s1 = todo_convert_stream_to_tbinned_stream(s1, range);
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
Ok(Box::pin(ret))
}
fn setup_from_higher_res_prebinned(
&mut self,
range: PreBinnedPatchRange,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<<ETB as EventsTimeBinner>::Output>> + Send>>, Error> {
) -> Result<
Pin<Box<dyn Stream<Item = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>> + Send>>,
Error,
> {
let g = self.query.patch().bin_t_len();
let h = range.grid_spec.bin_t_len();
trace!(
@@ -216,7 +222,7 @@ where
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType,
<ETB as EventsTimeBinner>::Output: Serialize + ReadableFromFile + 'static,
{
type Item = Sitemty<<ETB as EventsTimeBinner>::Output>;
type Item = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
@@ -290,7 +296,8 @@ where
self.values.len(),
);
self.streamlog.append(Level::INFO, msg);
let emp = <<ETB as EventsTimeBinner>::Output as Appendable>::empty();
// TODO use alias vias trait:
let emp = <<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output as Appendable>::empty();
let values = std::mem::replace(&mut self.values, emp);
let fut = write_pb_cache_min_max_avg_scalar(
values,
@@ -344,7 +351,7 @@ where
Ok(file) => {
self.read_from_cache = true;
let fut =
<<ETB as EventsTimeBinner>::Output as ReadableFromFile>::read_from_file(file)?;
<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output as ReadableFromFile>::read_from_file(file)?;
self.read_cache_fut = Some(Box::pin(fut));
continue 'outer;
}

5
disk/src/binned/prebinned.rs
View File

@@ -1,4 +1,6 @@
use crate::agg::binnedt4::{DefaultBinsTimeBinner, DefaultScalarEventsTimeBinner, DefaultSingleXBinTimeBinner};
use crate::agg::binnedt4::{
DefaultBinsTimeBinner, DefaultScalarEventsTimeBinner, DefaultSingleXBinTimeBinner, TimeBinnableType,
};
use crate::agg::enp::{Identity, WaveXBinner};
use crate::agg::streams::{Appendable, StreamItem};
use crate::binned::pbv2::{
@@ -41,6 +43,7 @@ where
<ETB as EventsTimeBinner>::Output: Serialize + ReadableFromFile + 'static,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<ETB as EventsTimeBinner>::Output>: Framable,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: Framable,
{
// TODO
// Currently, this mod uses stuff from pbv2, therefore complete path:

5
disk/src/binnedstream.rs
View File

@@ -12,6 +12,7 @@ use std::future::ready;
use std::pin::Pin;
use std::task::{Context, Poll};
// TODO remove after refactor.
pub struct BinnedScalarStreamFromPreBinnedPatches<SK>
where
SK: StreamKind,
@@ -113,8 +114,7 @@ where
}
}
// TODO change name, type is generic now:
// Can I remove the whole type or keep for static check?
// TODO remove after SK no longer needed.
impl<SK> Stream for BinnedScalarStreamFromPreBinnedPatches<SK>
where
SK: StreamKind,
@@ -126,6 +126,7 @@ where
}
}
// TODO remove after refactor.
pub struct BoxedStream<I> {
inp: Pin<Box<dyn Stream<Item = I> + Send>>,
}

1
disk/src/cache.rs
View File

@@ -25,6 +25,7 @@ use tokio::io::{AsyncRead, ReadBuf};
pub mod pbvfs;
// TODO move to a better fitting module:
pub struct HttpBodyAsAsyncRead {
inp: Response<Body>,
left: Bytes,

3
disk/src/cache/pbvfs.rs vendored
View File

@@ -13,6 +13,7 @@ use netpod::{NodeConfigCached, PerfOpts};
use std::pin::Pin;
use std::task::{Context, Poll};
// TODO remove when SK no longer needed.
pub struct PreBinnedScalarValueFetchedStream<SK>
where
SK: StreamKind,
@@ -75,7 +76,7 @@ where
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(item) => {
match decode_frame::<Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, Error>>(
&item, <Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, Error> as FrameType>::FRAME_TYPE_ID,
&item,
) {
Ok(Ok(item)) => Ready(Some(Ok(item))),
Ok(Err(e)) => {

104
disk/src/decode.rs
View File

@@ -1,7 +1,10 @@
use crate::agg::binnedt4::{TimeBinnableType, TimeBinnableTypeAggregator};
use crate::agg::enp::{Identity, WaveXBinner};
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::{Appendable, StreamItem};
use crate::binned::{
EventsNodeProcessor, NumOps, PushableIndex, RangeCompletableItem, RangeOverlapInfo, WithLen, WithTimestamps,
EventsNodeProcessor, FilterFittingInside, MinMaxAvgAggregator, MinMaxAvgBins, MinMaxAvgBinsAggregator, NumOps,
PushableIndex, RangeCompletableItem, RangeOverlapInfo, ReadPbv, ReadableFromFile, WithLen, WithTimestamps,
};
use crate::eventblobs::EventBlobsComplete;
use crate::eventchunker::EventFull;
@@ -11,11 +14,13 @@ use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::NanoRange;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;
use std::mem::size_of;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::fs::File;
pub trait Endianness: Send + Unpin {}
pub struct LittleEndian {}
@@ -135,7 +140,8 @@ impl<NTY, END> EventValueShape<NTY, END> for EventValuesDim0Case<NTY>
where
NTY: NumOps + NumFromBytes<NTY, END>,
{
type NumXAggToSingleBin = ProcAA<NTY>;
type NumXAggToSingleBin = Identity<NTY>;
// TODO:
type NumXAggToNBins = ProcAA<NTY>;
}
@@ -160,6 +166,7 @@ pub struct ProcBB<NTY> {
_m1: PhantomData<NTY>,
}
// TODO still in use or can go away?
#[derive(Serialize, Deserialize)]
pub struct MinMaxAvgScalarEventBatchGen<NTY> {
pub tss: Vec<u64>,
@@ -168,7 +175,10 @@ pub struct MinMaxAvgScalarEventBatchGen<NTY> {
pub avgs: Vec<Option<f32>>,
}
impl<NTY> MinMaxAvgScalarEventBatchGen<NTY> {
impl<NTY> MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
pub fn empty() -> Self {
Self {
tss: vec![],
@@ -179,12 +189,63 @@ impl<NTY> MinMaxAvgScalarEventBatchGen<NTY> {
}
}
impl<NTY> WithTimestamps for MinMaxAvgScalarEventBatchGen<NTY> {
impl<NTY> WithTimestamps for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn ts(&self, ix: usize) -> u64 {
self.tss[ix]
}
}
impl<NTY> FilterFittingInside for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
todo!()
}
}
impl<NTY> WithLen for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn len(&self) -> usize {
todo!()
}
}
impl<NTY> Appendable for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn empty() -> Self {
todo!()
}
fn append(&mut self, src: &Self) {
todo!()
}
}
impl<NTY> RangeOverlapInfo for MinMaxAvgScalarEventBatchGen<NTY>
where
NTY: NumOps,
{
fn ends_before(&self, range: NanoRange) -> bool {
todo!()
}
fn ends_after(&self, range: NanoRange) -> bool {
todo!()
}
fn starts_after(&self, range: NanoRange) -> bool {
todo!()
}
}
impl<NTY> Framable for Result<StreamItem<RangeCompletableItem<MinMaxAvgScalarEventBatchGen<NTY>>>, Error>
where
NTY: NumOps + Serialize,
@@ -210,11 +271,11 @@ impl<NTY, END> EventValueShape<NTY, END> for EventValuesDim1Case<NTY>
where
NTY: NumOps + NumFromBytes<NTY, END>,
{
type NumXAggToSingleBin = ProcBB<NTY>;
type NumXAggToSingleBin = WaveXBinner<NTY>;
// TODO:
type NumXAggToNBins = ProcBB<NTY>;
}
// TODO why not Serialize?
// TODO add pulse.
// TODO change name, it's not only about values, but more like batch of whole events.
#[derive(Serialize, Deserialize)]
@@ -284,6 +345,12 @@ impl<VT> RangeOverlapInfo for EventValues<VT> {
}
}
impl<VT> FilterFittingInside for EventValues<VT> {
fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
todo!()
}
}
impl<NTY> PushableIndex for EventValues<NTY>
where
NTY: NumOps,
@@ -308,6 +375,31 @@ where
}
}
impl<NTY> ReadableFromFile for EventValues<NTY>
where
NTY: NumOps,
{
fn read_from_file(file: File) -> Result<ReadPbv<Self>, Error> {
todo!()
}
fn from_buf(buf: &[u8]) -> Result<Self, Error> {
todo!()
}
}
impl<NTY> TimeBinnableType for EventValues<NTY>
where
NTY: NumOps,
{
type Output = MinMaxAvgBins<NTY>;
type Aggregator = MinMaxAvgAggregator<NTY>;
fn aggregator(range: NanoRange) -> Self::Aggregator {
Self::Aggregator::new(range)
}
}
pub struct EventsDecodedStream<NTY, END, EVS>
where
NTY: NumOps + NumFromBytes<NTY, END>,

6
disk/src/frame/makeframe.rs
View File

@@ -78,14 +78,14 @@ impl FrameType for Sitemty<MinMaxAvgScalarEventBatch> {
impl<NTY> FrameType for Sitemty<MinMaxAvgScalarEventBatchGen<NTY>>
where
NTY: SubFrId,
NTY: NumOps,
{
const FRAME_TYPE_ID: u32 = 0x400 + NTY::SUB;
}
impl<NTY> FrameType for Sitemty<EventValues<NTY>>
where
NTY: SubFrId,
NTY: NumOps,
{
const FRAME_TYPE_ID: u32 = 0x500 + NTY::SUB;
}
@@ -197,7 +197,7 @@ pub fn make_term_frame() -> BytesMut {
buf
}
pub fn decode_frame<T>(frame: &InMemoryFrame, frame_type: u32) -> Result<T, Error>
pub fn decode_frame<T>(frame: &InMemoryFrame) -> Result<T, Error>
where
T: FrameType + DeserializeOwned,
{

2
disk/src/merge.rs
View File

@@ -11,7 +11,7 @@ use std::collections::VecDeque;
use std::pin::Pin;
use std::task::{Context, Poll};
pub mod mergefromremote;
pub mod mergedfromremotes;
enum MergedCurVal<T> {
None,

0
disk/src/merge/mergefromremote.rs → disk/src/merge/mergedfromremotes.rs
View File

4
disk/src/raw.rs
View File

@@ -9,7 +9,7 @@ use crate::agg::streams::StreamItem;
use crate::binned::{EventsNodeProcessor, RangeCompletableItem, StreamKind};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::{make_frame, make_term_frame, FrameType};
use crate::raw::bffr::EventsFromFrames;
use crate::raw::eventsfromframes::EventsFromFrames;
use crate::Sitemty;
use err::Error;
use futures_core::Stream;
@@ -21,8 +21,8 @@ use tokio::net::TcpStream;
#[allow(unused_imports)]
use tracing::{debug, error, info, span, trace, warn, Level};
pub mod bffr;
pub mod conn;
pub mod eventsfromframes;
/**
Query parameters to request (optionally) X-processed, but not T-processed events.

20
disk/src/raw/conn.rs
View File

@@ -1,4 +1,5 @@
use crate::agg::binnedx::IntoBinnedXBins1;
use crate::agg::enp::{Identity, WaveXBinner};
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::streams::StreamItem;
use crate::agg::IntoDim1F32Stream;
@@ -104,6 +105,9 @@ fn make_num_pipeline_stream_evs<NTY, END, EVS, ENP>(
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
// TODO
// Can this work?
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Output>,
Sitemty<<ENP as EventsNodeProcessor>::Output>: Framable + 'static,
@@ -136,12 +140,14 @@ macro_rules! pipe4 {
$end,
$evs<$nty>,
<$evs<$nty> as EventValueShape<$nty, $end>>::NumXAggToSingleBin,
//Identity<$nty>,
>($evsv, $event_blobs),
AggKind::DimXBinsN(_) => make_num_pipeline_stream_evs::<
$nty,
$end,
$evs<$nty>,
<$evs<$nty> as EventValueShape<$nty, $end>>::NumXAggToSingleBin,
//WaveXBinner<$nty>,
>($evsv, $event_blobs),
}
};
@@ -161,6 +167,9 @@ macro_rules! pipe3 {
)
}
Shape::Wave(n) => {
// TODO
// Issue is that I try to generate too many combinations.
// e.g. I try to generic code for the combination of Shape::Scalar with WaveXBinner which does not match.
pipe4!(
$nty,
$end,
@@ -229,8 +238,7 @@ async fn events_conn_handler_inner_try(
error!("missing command frame");
return Err((Error::with_msg("missing command frame"), netout))?;
}
let frame_type = <EventQueryJsonStringFrame as FrameType>::FRAME_TYPE_ID;
let qitem: EventQueryJsonStringFrame = match decode_frame(&frames[0], frame_type) {
let qitem: EventQueryJsonStringFrame = match decode_frame(&frames[0]) {
Ok(k) => k,
Err(e) => return Err((e, netout).into()),
};
@@ -275,7 +283,7 @@ async fn events_conn_handler_inner_try(
compression: entry.is_compressed,
};
if false {
if true {
// TODO use a requested buffer size
let buffer_size = 1024 * 4;
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
@@ -288,10 +296,6 @@ async fn events_conn_handler_inner_try(
event_chunker_conf,
);
let shape = entry.to_shape().unwrap();
// TODO
// First, generalize over the number types.
// Then return boxed trait objects from the stream which are MakeFrame.
// The writeout does not need to be generic.
let mut p1 = pipe1!(entry.scalar_type, entry.byte_order, shape, evq.agg_kind, event_blobs);
while let Some(item) = p1.next().await {
let item = item.make_frame();
@@ -316,7 +320,7 @@ async fn events_conn_handler_inner_try(
}
Ok(())
} else {
// TODO use a requested buffer size
// TODO remove this scope after refactor.
let buffer_size = 1024 * 4;
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let s1 = EventBlobsComplete::new(

12
disk/src/raw/bffr.rs → disk/src/raw/eventsfromframes.rs
View File

@@ -13,7 +13,6 @@ use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncRead;
// TODO remove usage of SK, no longer needed.
pub struct EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
@@ -21,7 +20,7 @@ where
inp: InMemoryFrameAsyncReadStream<T>,
errored: bool,
completed: bool,
_m2: PhantomData<I>,
_m1: PhantomData<I>,
}
impl<T, I> EventsFromFrames<T, I>
@@ -33,7 +32,7 @@ where
inp,
errored: false,
completed: false,
_m2: PhantomData,
_m1: PhantomData,
}
}
}
@@ -41,10 +40,7 @@ where
impl<T, I> Stream for EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
//SK: StreamKind,
I: DeserializeOwned + Unpin,
// TODO see binned.rs better to express it on trait?
//Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>: FrameType,
Sitemty<I>: FrameType,
{
type Item = Sitemty<I>;
@@ -53,7 +49,7 @@ where
use Poll::*;
loop {
break if self.completed {
panic!("EventsFromFrames poll_next on completed");
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
Ready(None)
@@ -62,7 +58,7 @@ where
Ready(Some(Ok(item))) => match item {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(frame) => match decode_frame::<Sitemty<I>>(&frame, 0) {
StreamItem::DataItem(frame) => match decode_frame::<Sitemty<I>>(&frame) {
Ok(item) => match item {
Ok(item) => Ready(Some(Ok(item))),
Err(e) => {