daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First stats on console

Browse Source
This commit is contained in:
Dominik Werder
2021-05-01 18:38:21 +02:00
parent 0eb6b3364d
commit a47ffc843d
8 changed files with 99 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
disk/src/agg/binnedt.rs
View File

@@ -105,7 +105,8 @@ where
Ready(Some(Ok(ret)))
} else {
//info!("INGEST");
ag.ingest(&k);
let mut k = k;
ag.ingest(&mut k);
// if this input contains also data after the current bin, then I need to keep
// it for the next round.
if ag.ends_after(&k) {

41
disk/src/agg/eventbatch.rs
View File

@@ -1,8 +1,9 @@
use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
use crate::agg::{AggregatableTdim, AggregatableXdim1Bin, AggregatorTdim};
use crate::agg::{AggregatableTdim, AggregatableXdim1Bin, AggregatorTdim, ValuesExtractStats};
use bytes::{BufMut, Bytes, BytesMut};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::EventDataReadStats;
use serde::{Deserialize, Serialize};
use std::mem::size_of;
@@ -12,6 +13,8 @@ pub struct MinMaxAvgScalarEventBatch {
pub mins: Vec<f32>,
pub maxs: Vec<f32>,
pub avgs: Vec<f32>,
pub event_data_read_stats: EventDataReadStats,
pub values_extract_stats: ValuesExtractStats,
}
impl MinMaxAvgScalarEventBatch {
@@ -21,6 +24,8 @@ impl MinMaxAvgScalarEventBatch {
mins: vec![],
maxs: vec![],
avgs: vec![],
event_data_read_stats: EventDataReadStats::new(),
values_extract_stats: ValuesExtractStats::new(),
}
}
#[allow(dead_code)]
@@ -85,12 +90,14 @@ impl std::fmt::Debug for MinMaxAvgScalarEventBatch {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
fmt,
"MinMaxAvgScalarEventBatch count {} tss {:?} mins {:?} maxs {:?} avgs {:?}",
"MinMaxAvgScalarEventBatch count {} tss {:?} mins {:?} maxs {:?} avgs {:?} EDS {:?} VXS {:?}",
self.tss.len(),
self.tss,
self.mins,
self.maxs,
self.avgs,
self.event_data_read_stats,
self.values_extract_stats,
)
}
}
@@ -142,6 +149,8 @@ pub struct MinMaxAvgScalarEventBatchAggregator {
min: f32,
max: f32,
sum: f32,
event_data_read_stats: EventDataReadStats,
values_extract_stats: ValuesExtractStats,
}
impl MinMaxAvgScalarEventBatchAggregator {
@@ -153,6 +162,8 @@ impl MinMaxAvgScalarEventBatchAggregator {
max: f32::MIN,
sum: 0f32,
count: 0,
event_data_read_stats: EventDataReadStats::new(),
values_extract_stats: ValuesExtractStats::new(),
}
}
}
@@ -182,15 +193,19 @@ impl AggregatorTdim for MinMaxAvgScalarEventBatchAggregator {
}
}
fn ingest(&mut self, v: &Self::InputValue) {
trace!(
"ingest {} {} {} {:?} {:?}",
self.ends_before(v),
self.ends_after(v),
self.starts_after(v),
v.tss.first().map(|k| k / SEC),
v.tss.last().map(|k| k / SEC),
);
fn ingest(&mut self, v: &mut Self::InputValue) {
if false {
trace!(
"ingest {} {} {} {:?} {:?}",
self.ends_before(v),
self.ends_after(v),
self.starts_after(v),
v.tss.first().map(|k| k / SEC),
v.tss.last().map(|k| k / SEC),
);
}
self.event_data_read_stats.trans(&mut v.event_data_read_stats);
self.values_extract_stats.trans(&mut v.values_extract_stats);
for i1 in 0..v.tss.len() {
let ts = v.tss[i1];
if ts < self.ts1 {
@@ -227,7 +242,7 @@ impl AggregatorTdim for MinMaxAvgScalarEventBatchAggregator {
}
}
fn result(self) -> Self::OutputValue {
fn result(mut self) -> Self::OutputValue {
let min = if self.min == f32::MAX { f32::NAN } else { self.min };
let max = if self.max == f32::MIN { f32::NAN } else { self.max };
let avg = if self.count == 0 {
@@ -242,6 +257,8 @@ impl AggregatorTdim for MinMaxAvgScalarEventBatchAggregator {
mins: vec![min],
maxs: vec![max],
avgs: vec![avg],
event_data_read_stats: std::mem::replace(&mut self.event_data_read_stats, EventDataReadStats::new()),
values_extract_stats: std::mem::replace(&mut self.values_extract_stats, ValuesExtractStats::new()),
}
}
}

26
disk/src/agg/scalarbinbatch.rs
View File

@@ -1,8 +1,8 @@
use crate::agg::{AggregatableTdim, AggregatableXdim1Bin, AggregatorTdim, Fits, FitsInside};
use crate::agg::{AggregatableTdim, AggregatableXdim1Bin, AggregatorTdim, Fits, FitsInside, ValuesExtractStats};
use bytes::{BufMut, Bytes, BytesMut};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::NanoRange;
use netpod::{EventDataReadStats, NanoRange};
use serde::{Deserialize, Serialize};
use std::mem::size_of;
@@ -15,6 +15,8 @@ pub struct MinMaxAvgScalarBinBatch {
pub mins: Vec<f32>,
pub maxs: Vec<f32>,
pub avgs: Vec<f32>,
pub event_data_read_stats: EventDataReadStats,
pub values_extract_stats: ValuesExtractStats,
}
impl MinMaxAvgScalarBinBatch {
@@ -26,6 +28,8 @@ impl MinMaxAvgScalarBinBatch {
mins: vec![],
maxs: vec![],
avgs: vec![],
event_data_read_stats: EventDataReadStats::new(),
values_extract_stats: ValuesExtractStats::new(),
}
}
@@ -168,12 +172,14 @@ impl std::fmt::Debug for MinMaxAvgScalarBinBatch {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
fmt,
"MinMaxAvgScalarBinBatch count {} ts1s {:?} ts2s {:?} counts {:?} avgs {:?}",
"MinMaxAvgScalarBinBatch count {} ts1s {:?} ts2s {:?} counts {:?} avgs {:?} EDS {:?} VXS {:?}",
self.ts1s.len(),
self.ts1s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.ts2s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.counts,
self.avgs
self.avgs,
self.event_data_read_stats,
self.values_extract_stats,
)
}
}
@@ -201,6 +207,8 @@ pub struct MinMaxAvgScalarBinBatchAggregator {
max: f32,
sum: f32,
sumc: u64,
event_data_read_stats: EventDataReadStats,
values_extract_stats: ValuesExtractStats,
}
impl MinMaxAvgScalarBinBatchAggregator {
@@ -213,6 +221,8 @@ impl MinMaxAvgScalarBinBatchAggregator {
max: f32::MIN,
sum: 0f32,
sumc: 0,
event_data_read_stats: EventDataReadStats::new(),
values_extract_stats: ValuesExtractStats::new(),
}
}
}
@@ -242,7 +252,9 @@ impl AggregatorTdim for MinMaxAvgScalarBinBatchAggregator {
}
}
fn ingest(&mut self, v: &Self::InputValue) {
fn ingest(&mut self, v: &mut Self::InputValue) {
self.event_data_read_stats.trans(&mut v.event_data_read_stats);
self.values_extract_stats.trans(&mut v.values_extract_stats);
for i1 in 0..v.ts1s.len() {
let ts1 = v.ts1s[i1];
let ts2 = v.ts2s[i1];
@@ -260,7 +272,7 @@ impl AggregatorTdim for MinMaxAvgScalarBinBatchAggregator {
}
}
fn result(self) -> Self::OutputValue {
fn result(mut self) -> Self::OutputValue {
let min = if self.min == f32::MAX { f32::NAN } else { self.min };
let max = if self.max == f32::MIN { f32::NAN } else { self.max };
let avg = if self.sumc == 0 {
@@ -275,6 +287,8 @@ impl AggregatorTdim for MinMaxAvgScalarBinBatchAggregator {
mins: vec![min],
maxs: vec![max],
avgs: vec![avg],
event_data_read_stats: std::mem::replace(&mut self.event_data_read_stats, EventDataReadStats::new()),
values_extract_stats: std::mem::replace(&mut self.values_extract_stats, ValuesExtractStats::new()),
}
}
}