daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2021-06-15 17:19:47 +02:00
parent bebce14f56
commit edafc610c2
15 changed files with 613 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
disk/src/binned/binnedfrompbv.rs
+6 -2
View File
@@ -11,7 +11,9 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use http::{StatusCode, Uri};
use netpod::log::*;
use netpod::{AggKind, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts, PreBinnedPatchIterator};
use netpod::{
x_bin_count, AggKind, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, Shape,
};
use serde::de::DeserializeOwned;
use std::future::ready;
use std::marker::PhantomData;
@@ -165,6 +167,7 @@ where
patch_it: PreBinnedPatchIterator,
channel: Channel,
range: BinnedRange,
shape: Shape,
agg_kind: AggKind,
cache_usage: CacheUsage,
node_config: &NodeConfigCached,
@@ -184,6 +187,7 @@ where
let pmax = patches.len();
let inp = futures_util::stream::iter(patches.into_iter().enumerate())
.map({
let agg_kind = agg_kind.clone();
let node_config = node_config.clone();
move |(pix, patch)| {
let query = PreBinnedQuery::new(
@@ -235,7 +239,7 @@ where
ready(g)
}
});
let inp = TBinnerStream::<_, TBT>::new(inp, range);
let inp = TBinnerStream::<_, TBT>::new(inp, range, x_bin_count(&shape, &agg_kind));
Ok(Self {
inp: Box::pin(inp),
_m1: PhantomData,
disk/src/binned/dim1.rs
+5 -3
View File
@@ -8,6 +8,7 @@ use crate::binned::{
use crate::Sitemty;
use chrono::{TimeZone, Utc};
use err::Error;
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
use num_traits::Zero;
@@ -173,8 +174,8 @@ where
type Output = MinMaxAvgDim1Bins<NTY>;
type Aggregator = MinMaxAvgDim1BinsAggregator<NTY>;
fn aggregator(range: NanoRange, bin_count: usize) -> Self::Aggregator {
Self::Aggregator::new(range)
fn aggregator(range: NanoRange, x_bin_count: usize) -> Self::Aggregator {
Self::Aggregator::new(range, x_bin_count)
}
}
@@ -317,7 +318,7 @@ pub struct MinMaxAvgDim1BinsAggregator<NTY> {
}
impl<NTY> MinMaxAvgDim1BinsAggregator<NTY> {
pub fn new(range: NanoRange, bin_count: usize) -> Self {
pub fn new(range: NanoRange, _x_bin_count: usize) -> Self {
Self {
range,
count: 0,
@@ -439,6 +440,7 @@ pub struct WaveEventsCollector<NTY> {
impl<NTY> WaveEventsCollector<NTY> {
pub fn new(_bin_count_exp: u32) -> Self {
info!("\n\nWaveEventsCollector\n\n");
Self {
vals: WaveEvents::empty(),
range_complete: false,
disk/src/binned/pbv.rs
+13 -3
View File
@@ -14,7 +14,9 @@ use err::Error;
use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use netpod::log::*;
use netpod::{BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange};
use netpod::{
x_bin_count, AggKind, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, Shape,
};
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::future::Future;
@@ -33,6 +35,8 @@ where
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Output>,
{
query: PreBinnedQuery,
shape: Shape,
agg_kind: AggKind,
node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<File, io::Error>> + Send>>>,
fut2: Option<
@@ -73,9 +77,11 @@ where
// TODO who exactly needs this DeserializeOwned?
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: FrameType + DeserializeOwned,
{
pub fn new(query: PreBinnedQuery, node_config: &NodeConfigCached) -> Self {
pub fn new(query: PreBinnedQuery, shape: Shape, agg_kind: AggKind, node_config: &NodeConfigCached) -> Self {
Self {
query,
shape,
agg_kind,
node_config: node_config.clone(),
open_check_local_file: None,
fut2: None,
@@ -124,7 +130,11 @@ where
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(
s,
range,
x_bin_count(&self.shape, &self.agg_kind),
);
Ok(Box::pin(ret))
}
disk/src/binned/prebinned.rs
+81 -26
View File
@@ -1,5 +1,4 @@
use crate::agg::binnedt::TimeBinnableType;
use crate::agg::enp::{Identity, WaveXBinner};
use crate::agg::streams::Appendable;
use crate::binned::pbv::PreBinnedValueStream;
use crate::binned::query::PreBinnedQuery;
@@ -15,14 +14,17 @@ use bytes::Bytes;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::{ByteOrder, NodeConfigCached, ScalarType, Shape};
use netpod::{AggKind, ByteOrder, NodeConfigCached, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::pin::Pin;
fn make_num_pipeline_nty_end_evs_enp<NTY, END, EVS, ENP>(
shape: Shape,
agg_kind: AggKind,
_event_value_shape: EVS,
_events_node_proc: ENP,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>
@@ -36,13 +38,14 @@ where
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
Framable + FrameType + DeserializeOwned,
{
let ret = PreBinnedValueStream::<NTY, END, EVS, ENP>::new(query, node_config);
let ret = PreBinnedValueStream::<NTY, END, EVS, ENP>::new(query, shape, agg_kind, node_config);
let ret = StreamExt::map(ret, |item| Box::new(item) as Box<dyn Framable>);
Box::pin(ret)
}
fn make_num_pipeline_nty_end<NTY, END>(
shape: Shape,
agg_kind: AggKind,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>
@@ -51,24 +54,74 @@ where
END: Endianness + 'static,
{
match shape {
Shape::Scalar => make_num_pipeline_nty_end_evs_enp::<NTY, END, _, Identity<NTY>>(
EventValuesDim0Case::new(),
query,
node_config,
),
Shape::Wave(n) => make_num_pipeline_nty_end_evs_enp::<NTY, END, _, WaveXBinner<NTY>>(
EventValuesDim1Case::new(n),
query,
node_config,
),
Shape::Scalar => {
let evs = EventValuesDim0Case::new();
match agg_kind {
AggKind::DimXBins1 => {
let events_node_proc = <<EventValuesDim0Case<NTY> as EventValueShape<NTY, END>>::NumXAggToSingleBin as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
make_num_pipeline_nty_end_evs_enp::<NTY, END, _, _>(
shape,
agg_kind,
evs,
events_node_proc,
query,
node_config,
)
}
AggKind::DimXBinsN(_) => {
let events_node_proc = <<EventValuesDim0Case<NTY> as EventValueShape<NTY, END>>::NumXAggToNBins as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
make_num_pipeline_nty_end_evs_enp::<NTY, END, _, _>(
shape,
agg_kind,
evs,
events_node_proc,
query,
node_config,
)
}
AggKind::Plain => {
panic!();
}
}
}
Shape::Wave(n) => {
let evs = EventValuesDim1Case::new(n);
match agg_kind {
AggKind::DimXBins1 => {
let events_node_proc = <<EventValuesDim1Case<NTY> as EventValueShape<NTY, END>>::NumXAggToSingleBin as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
make_num_pipeline_nty_end_evs_enp::<NTY, END, _, _>(
shape,
agg_kind,
evs,
events_node_proc,
query,
node_config,
)
}
AggKind::DimXBinsN(_) => {
let events_node_proc = <<EventValuesDim1Case<NTY> as EventValueShape<NTY, END>>::NumXAggToNBins as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
make_num_pipeline_nty_end_evs_enp::<NTY, END, _, _>(
shape,
agg_kind,
evs,
events_node_proc,
query,
node_config,
)
}
AggKind::Plain => {
panic!();
}
}
}
}
}
macro_rules! match_end {
($nty:ident, $end:expr, $shape:expr, $query:expr, $node_config:expr) => {
($nty:ident, $end:expr, $shape:expr, $agg_kind:expr, $query:expr, $node_config:expr) => {
match $end {
ByteOrder::LE => make_num_pipeline_nty_end::<$nty, LittleEndian>($shape, $query, $node_config),
ByteOrder::BE => make_num_pipeline_nty_end::<$nty, BigEndian>($shape, $query, $node_config),
ByteOrder::LE => make_num_pipeline_nty_end::<$nty, LittleEndian>($shape, $agg_kind, $query, $node_config),
ByteOrder::BE => make_num_pipeline_nty_end::<$nty, BigEndian>($shape, $agg_kind, $query, $node_config),
}
};
}
@@ -77,20 +130,21 @@ fn make_num_pipeline(
scalar_type: ScalarType,
byte_order: ByteOrder,
shape: Shape,
agg_kind: AggKind,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>> {
match scalar_type {
ScalarType::U8 => match_end!(u8, byte_order, shape, query, node_config),
ScalarType::U16 => match_end!(u16, byte_order, shape, query, node_config),
ScalarType::U32 => match_end!(u32, byte_order, shape, query, node_config),
ScalarType::U64 => match_end!(u64, byte_order, shape, query, node_config),
ScalarType::I8 => match_end!(i8, byte_order, shape, query, node_config),
ScalarType::I16 => match_end!(i16, byte_order, shape, query, node_config),
ScalarType::I32 => match_end!(i32, byte_order, shape, query, node_config),
ScalarType::I64 => match_end!(i64, byte_order, shape, query, node_config),
ScalarType::F32 => match_end!(f32, byte_order, shape, query, node_config),
ScalarType::F64 => match_end!(f64, byte_order, shape, query, node_config),
ScalarType::U8 => match_end!(u8, byte_order, shape, agg_kind, query, node_config),
ScalarType::U16 => match_end!(u16, byte_order, shape, agg_kind, query, node_config),
ScalarType::U32 => match_end!(u32, byte_order, shape, agg_kind, query, node_config),
ScalarType::U64 => match_end!(u64, byte_order, shape, agg_kind, query, node_config),
ScalarType::I8 => match_end!(i8, byte_order, shape, agg_kind, query, node_config),
ScalarType::I16 => match_end!(i16, byte_order, shape, agg_kind, query, node_config),
ScalarType::I32 => match_end!(i32, byte_order, shape, agg_kind, query, node_config),
ScalarType::I64 => match_end!(i64, byte_order, shape, agg_kind, query, node_config),
ScalarType::F32 => match_end!(f32, byte_order, shape, agg_kind, query, node_config),
ScalarType::F64 => match_end!(f64, byte_order, shape, agg_kind, query, node_config),
}
}
@@ -137,6 +191,7 @@ pub async fn pre_binned_bytes_for_http(
entry.scalar_type.clone(),
entry.byte_order.clone(),
entry.to_shape()?,
query.agg_kind().clone(),
query.clone(),
node_config,
)