daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP not compile

Browse Source
This commit is contained in:
Dominik Werder
2021-05-26 09:08:44 +02:00
parent a76e86e623
commit 2b1be2f2b9
11 changed files with 235 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
disk/src/agg/binnedt.rs
View File

@@ -1,6 +1,6 @@
use crate::agg::streams::StreamItem;
use crate::agg::AggregatableXdim1Bin;
use crate::binned::RangeCompletableItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
@@ -10,9 +10,12 @@ use std::collections::VecDeque;
use std::pin::Pin;
use std::task::{Context, Poll};
pub trait AggregatorTdim: Sized + Unpin {
pub trait AggregatorTdim<SK>: Sized + Unpin
where
SK: BinnedStreamKind,
{
type InputValue;
type OutputValue: AggregatableXdim1Bin + AggregatableTdim + Unpin;
type OutputValue: AggregatableXdim1Bin<SK> + AggregatableTdim<SK> + Unpin;
fn ends_before(&self, inp: &Self::InputValue) -> bool;
fn ends_after(&self, inp: &Self::InputValue) -> bool;
fn starts_after(&self, inp: &Self::InputValue) -> bool;
@@ -20,34 +23,44 @@ pub trait AggregatorTdim: Sized + Unpin {
fn result(self) -> Vec<Self::OutputValue>;
}
pub trait AggregatableTdim: Sized {
type Output: AggregatableXdim1Bin + AggregatableTdim;
type Aggregator: AggregatorTdim<InputValue = Self>;
pub trait AggregatableTdim<SK>: Sized
where
SK: BinnedStreamKind,
{
//type Output: AggregatableXdim1Bin + AggregatableTdim;
type Aggregator: AggregatorTdim<SK, InputValue = Self, OutputValue = <SK as BinnedStreamKind>::TBinnedBins>;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator;
}
pub trait IntoBinnedT {
type StreamOut: Stream;
pub trait IntoBinnedT<SK>
where
SK: BinnedStreamKind,
{
type StreamOut: Stream<
Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>,
>;
fn into_binned_t(self, spec: BinnedRange) -> Self::StreamOut;
}
impl<S, I> IntoBinnedT for S
impl<S, I, SK> IntoBinnedT<SK> for S
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableTdim + Unpin,
I: AggregatableTdim<SK> + Unpin,
I::Aggregator: Unpin,
{
type StreamOut = IntoBinnedTDefaultStream<S, I>;
type StreamOut = IntoBinnedTDefaultStream<S, I, SK>;
fn into_binned_t(self, spec: BinnedRange) -> Self::StreamOut {
IntoBinnedTDefaultStream::new(self, spec)
}
}
pub struct IntoBinnedTDefaultStream<S, I>
pub struct IntoBinnedTDefaultStream<S, I, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableTdim,
I: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
inp: S,
aggtor: Option<I::Aggregator>,
@@ -60,13 +73,15 @@ where
left: Option<Poll<Option<Result<StreamItem<RangeCompletableItem<I>>, Error>>>>,
errored: bool,
completed: bool,
tmp_agg_results: VecDeque<<I::Aggregator as AggregatorTdim>::OutputValue>,
tmp_agg_results: VecDeque<<I::Aggregator as AggregatorTdim<SK>>::OutputValue>,
_marker: std::marker::PhantomData<SK>,
}
impl<S, I> IntoBinnedTDefaultStream<S, I>
impl<S, I, SK> IntoBinnedTDefaultStream<S, I, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableTdim,
I: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
pub fn new(inp: S, spec: BinnedRange) -> Self {
let range = spec.get_range(0);
@@ -83,6 +98,7 @@ where
errored: false,
completed: false,
tmp_agg_results: VecDeque::new(),
_marker: std::marker::PhantomData::default(),
}
}
@@ -116,7 +132,9 @@ where
&mut self,
cur: Poll<Option<Result<StreamItem<RangeCompletableItem<I>>, Error>>>,
) -> Option<
Poll<Option<Result<StreamItem<RangeCompletableItem<<I::Aggregator as AggregatorTdim>::OutputValue>>, Error>>>,
Poll<
Option<Result<StreamItem<RangeCompletableItem<<I::Aggregator as AggregatorTdim<SK>>::OutputValue>>, Error>>,
>,
> {
use Poll::*;
match cur {
@@ -178,13 +196,15 @@ where
}
}
impl<S, I> Stream for IntoBinnedTDefaultStream<S, I>
impl<S, I, SK> Stream for IntoBinnedTDefaultStream<S, I, SK>
where
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableTdim + Unpin,
I: AggregatableTdim<SK> + Unpin,
I::Aggregator: Unpin,
SK: BinnedStreamKind,
{
type Item = Result<StreamItem<RangeCompletableItem<<I::Aggregator as AggregatorTdim>::OutputValue>>, Error>;
//type Item = Result<StreamItem<RangeCompletableItem<<I::Aggregator as AggregatorTdim>::OutputValue>>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::TBinnedBins>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;

84
disk/src/agg/binnedt2.rs
View File

@@ -213,29 +213,97 @@ where
}
}
pub struct MinMaxAvgScalarBinBatchAgg {}
pub struct MinMaxAvgScalarBinBatchAgg {
ts1: u64,
ts2: u64,
count: u64,
min: f32,
max: f32,
sum: f32,
sumc: u64,
}
impl MinMaxAvgScalarBinBatchAgg {
pub fn new(ts1: u64, ts2: u64) -> Self {
Self {
ts1,
ts2,
count: 0,
min: f32::MAX,
max: f32::MIN,
sum: 0f32,
sumc: 0,
}
}
}
impl AggregatorTdim2 for MinMaxAvgScalarBinBatchAgg {
type InputValue = MinMaxAvgScalarBinBatch;
fn ends_before(&self, inp: &Self::InputValue) -> bool {
todo!()
match inp.ts2s.last() {
Some(&ts) => ts <= self.ts1,
None => true,
}
}
fn ends_after(&self, inp: &Self::InputValue) -> bool {
todo!()
match inp.ts2s.last() {
Some(&ts) => ts >= self.ts2,
_ => panic!(),
}
}
fn starts_after(&self, inp: &Self::InputValue) -> bool {
todo!()
match inp.ts1s.first() {
Some(&ts) => ts >= self.ts2,
_ => panic!(),
}
}
fn ingest(&mut self, inp: &mut Self::InputValue) {
todo!()
fn ingest(&mut self, v: &mut Self::InputValue) {
for i1 in 0..v.ts1s.len() {
let ts1 = v.ts1s[i1];
let ts2 = v.ts2s[i1];
if ts2 <= self.ts1 {
continue;
} else if ts1 >= self.ts2 {
continue;
} else {
self.count += v.counts[i1];
self.min = self.min.min(v.mins[i1]);
self.max = self.max.max(v.maxs[i1]);
let x = v.avgs[i1];
if x.is_nan() {
} else {
if self.sum.is_nan() {
self.sum = x;
} else {
self.sum += x;
}
self.sumc += 1;
}
}
}
}
fn result(self) -> Vec<Self::InputValue> {
todo!()
let min = if self.min == f32::MAX { f32::NAN } else { self.min };
let max = if self.max == f32::MIN { f32::NAN } else { self.max };
let avg = if self.sumc == 0 {
f32::NAN
} else {
self.sum / self.sumc as f32
};
let v = MinMaxAvgScalarBinBatch {
ts1s: vec![self.ts1],
ts2s: vec![self.ts2],
counts: vec![self.count],
mins: vec![min],
maxs: vec![max],
avgs: vec![avg],
};
vec![v]
}
}
@@ -243,6 +311,6 @@ impl AggregatableTdim2 for MinMaxAvgScalarBinBatch {
type Aggregator = MinMaxAvgScalarBinBatchAgg;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator {
todo!()
Self::Aggregator::new(ts1, ts2)
}
}

30
disk/src/agg/binnedx.rs
View File

@@ -1,45 +1,53 @@
use crate::agg::streams::StreamItem;
use crate::agg::AggregatableXdim1Bin;
use crate::binned::RangeCompletableItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use std::pin::Pin;
use std::task::{Context, Poll};
pub trait IntoBinnedXBins1<I>
pub trait IntoBinnedXBins1<I, SK>
where
SK: BinnedStreamKind,
Self: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin,
I: AggregatableXdim1Bin<SK>,
{
type StreamOut;
fn into_binned_x_bins_1(self) -> Self::StreamOut;
}
impl<S, I> IntoBinnedXBins1<I> for S
impl<S, I, SK> IntoBinnedXBins1<I, SK> for S
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin,
I: AggregatableXdim1Bin<SK>,
{
type StreamOut = IntoBinnedXBins1DefaultStream<S, I>;
type StreamOut = IntoBinnedXBins1DefaultStream<S, I, SK>;
fn into_binned_x_bins_1(self) -> Self::StreamOut {
IntoBinnedXBins1DefaultStream { inp: self }
IntoBinnedXBins1DefaultStream {
inp: self,
_marker: std::marker::PhantomData::default(),
}
}
}
pub struct IntoBinnedXBins1DefaultStream<S, I>
pub struct IntoBinnedXBins1DefaultStream<S, I, SK>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin,
I: AggregatableXdim1Bin<SK>,
{
inp: S,
_marker: std::marker::PhantomData<SK>,
}
impl<S, I> Stream for IntoBinnedXBins1DefaultStream<S, I>
impl<S, I, SK> Stream for IntoBinnedXBins1DefaultStream<S, I, SK>
where
SK: BinnedStreamKind,
S: Stream<Item = Result<StreamItem<RangeCompletableItem<I>>, Error>> + Unpin,
I: AggregatableXdim1Bin,
I: AggregatableXdim1Bin<SK>,
{
type Item = Result<StreamItem<RangeCompletableItem<I::Output>>, Error>;

19
disk/src/agg/eventbatch.rs
View File

@@ -2,7 +2,7 @@ use crate::agg::binnedt::{AggregatableTdim, AggregatorTdim};
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::streams::StreamItem;
use crate::agg::AggregatableXdim1Bin;
use crate::binned::{MakeBytesFrame, RangeCompletableItem};
use crate::binned::{BinnedStreamKind, MakeBytesFrame, RangeCompletableItem};
use crate::frame::makeframe::make_frame;
use bytes::{BufMut, Bytes, BytesMut};
use err::Error;
@@ -100,15 +100,21 @@ impl std::fmt::Debug for MinMaxAvgScalarEventBatch {
}
}
impl AggregatableXdim1Bin for MinMaxAvgScalarEventBatch {
impl<SK> AggregatableXdim1Bin<SK> for MinMaxAvgScalarEventBatch
where
SK: BinnedStreamKind,
{
type Output = MinMaxAvgScalarEventBatch;
fn into_agg(self) -> Self::Output {
self
}
}
impl AggregatableTdim for MinMaxAvgScalarEventBatch {
type Output = MinMaxAvgScalarBinBatch;
impl<SK> AggregatableTdim<SK> for MinMaxAvgScalarEventBatch
where
SK: BinnedStreamKind,
{
//type Output = MinMaxAvgScalarBinBatch;
type Aggregator = MinMaxAvgScalarEventBatchAggregator;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator {
@@ -165,7 +171,10 @@ impl MinMaxAvgScalarEventBatchAggregator {
}
}
impl AggregatorTdim for MinMaxAvgScalarEventBatchAggregator {
impl<SK> AggregatorTdim<SK> for MinMaxAvgScalarEventBatchAggregator
where
SK: BinnedStreamKind,
{
type InputValue = MinMaxAvgScalarEventBatch;
type OutputValue = MinMaxAvgScalarBinBatch;

19
disk/src/agg/scalarbinbatch.rs
View File

@@ -1,7 +1,7 @@
use crate::agg::binnedt::{AggregatableTdim, AggregatorTdim};
use crate::agg::streams::{Bins, StreamItem};
use crate::agg::{AggregatableXdim1Bin, Fits, FitsInside};
use crate::binned::{MakeBytesFrame, RangeCompletableItem};
use crate::binned::{BinnedStreamKind, MakeBytesFrame, RangeCompletableItem};
use crate::frame::makeframe::make_frame;
use bytes::{BufMut, Bytes, BytesMut};
use err::Error;
@@ -185,15 +185,21 @@ impl MinMaxAvgScalarBinBatch {
}
}
impl AggregatableXdim1Bin for MinMaxAvgScalarBinBatch {
impl<SK> AggregatableXdim1Bin<SK> for MinMaxAvgScalarBinBatch
where
SK: BinnedStreamKind,
{
type Output = MinMaxAvgScalarBinBatch;
fn into_agg(self) -> Self::Output {
todo!()
}
}
impl AggregatableTdim for MinMaxAvgScalarBinBatch {
type Output = MinMaxAvgScalarBinBatch;
impl<SK> AggregatableTdim<SK> for MinMaxAvgScalarBinBatch
where
SK: BinnedStreamKind,
{
//type Output = MinMaxAvgScalarBinBatch;
type Aggregator = MinMaxAvgScalarBinBatchAggregator;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator {
@@ -231,7 +237,10 @@ impl MinMaxAvgScalarBinBatchAggregator {
}
}
impl AggregatorTdim for MinMaxAvgScalarBinBatchAggregator {
impl<SK> AggregatorTdim<SK> for MinMaxAvgScalarBinBatchAggregator
where
SK: BinnedStreamKind,
{
type InputValue = MinMaxAvgScalarBinBatch;
type OutputValue = MinMaxAvgScalarBinBatch;

38
disk/src/agg/streams.rs
View File

@@ -1,5 +1,6 @@
use crate::agg::binnedt::{AggregatableTdim, AggregatorTdim};
use crate::agg::AggregatableXdim1Bin;
use crate::binned::BinnedStreamKind;
use crate::streamlog::LogItem;
use err::Error;
use netpod::EventDataReadStats;
@@ -36,11 +37,12 @@ pub trait ToJsonResult {
fn to_json_result(&self) -> Result<Self::Output, Error>;
}
impl<T> AggregatableXdim1Bin for StreamItem<T>
impl<T, SK> AggregatableXdim1Bin<SK> for StreamItem<T>
where
T: AggregatableTdim + AggregatableXdim1Bin,
SK: BinnedStreamKind,
T: AggregatableTdim<SK> + AggregatableXdim1Bin<SK>,
{
type Output = StreamItem<<T as AggregatableXdim1Bin>::Output>;
type Output = StreamItem<<T as AggregatableXdim1Bin<SK>>::Output>;
fn into_agg(self) -> Self::Output {
match self {
@@ -51,30 +53,33 @@ where
}
}
pub struct StreamItemAggregator<T>
pub struct StreamItemAggregator<T, SK>
where
T: AggregatableTdim,
T: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
inner_agg: <T as AggregatableTdim>::Aggregator,
inner_agg: <T as AggregatableTdim<SK>>::Aggregator,
}
impl<T> StreamItemAggregator<T>
impl<T, SK> StreamItemAggregator<T, SK>
where
T: AggregatableTdim,
T: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
pub fn new(ts1: u64, ts2: u64) -> Self {
Self {
inner_agg: <T as AggregatableTdim>::aggregator_new_static(ts1, ts2),
inner_agg: <T as AggregatableTdim<SK>>::aggregator_new_static(ts1, ts2),
}
}
}
impl<T> AggregatorTdim for StreamItemAggregator<T>
impl<T, SK> AggregatorTdim<SK> for StreamItemAggregator<T, SK>
where
T: AggregatableTdim,
T: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
type InputValue = StreamItem<T>;
type OutputValue = StreamItem<<<T as AggregatableTdim>::Aggregator as AggregatorTdim>::OutputValue>;
type OutputValue = StreamItem<<<T as AggregatableTdim<SK>>::Aggregator as AggregatorTdim<SK>>::OutputValue>;
fn ends_before(&self, inp: &Self::InputValue) -> bool {
match inp {
@@ -119,12 +124,13 @@ where
}
}
impl<T> AggregatableTdim for StreamItem<T>
impl<T, SK> AggregatableTdim<SK> for StreamItem<T>
where
T: AggregatableTdim,
T: AggregatableTdim<SK>,
SK: BinnedStreamKind,
{
type Output = StreamItem<<StreamItemAggregator<T> as AggregatorTdim>::OutputValue>;
type Aggregator = StreamItemAggregator<T>;
//type Output = StreamItem<<StreamItemAggregator<T> as AggregatorTdim>::OutputValue>;
type Aggregator = StreamItemAggregator<T, SK>;
fn aggregator_new_static(ts1: u64, ts2: u64) -> Self::Aggregator {
Self::Aggregator::new(ts1, ts2)