daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deliver float events from scylla table

Browse Source
This commit is contained in:
Dominik Werder
2022-04-22 22:44:32 +02:00
parent 22b43fe012
commit e7016eda36
23 changed files with 917 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
items/src/lib.rs
View File

@@ -381,7 +381,7 @@ where
Self: Sized,
{
type Value;
fn append_event(ret: Option<Self>, ts: u64, value: Self::Value) -> Self;
fn append_event(ret: Option<Self>, ts: u64, pulse: u64, value: Self::Value) -> Self;
}
pub trait TimeBins: Send + Unpin + WithLen + Appendable + FilterFittingInside {
@@ -481,6 +481,12 @@ pub fn ts_offs_from_abs(tss: &[u64]) -> (u64, Vec<u64>, Vec<u64>) {
(ts_anchor_sec, ts_off_ms, ts_off_ns)
}
pub fn pulse_offs_from_abs(pulse: &[u64]) -> (u64, Vec<u64>) {
let pulse_anchor = pulse.first().map_or(0, |k| *k);
let pulse_off: Vec<_> = pulse.iter().map(|k| *k - pulse_anchor).collect();
(pulse_anchor, pulse_off)
}
pub trait TimeBinnableTypeAggregator: Send {
type Input: TimeBinnableType;
type Output: TimeBinnableType;

13
items/src/minmaxavgdim1bins.rs
View File

@@ -2,9 +2,9 @@ use crate::numops::NumOps;
use crate::streams::{Collectable, Collector, ToJsonBytes, ToJsonResult};
use crate::waveevents::WaveEvents;
use crate::{
ts_offs_from_abs, Appendable, FilterFittingInside, Fits, FitsInside, IsoDateTime, RangeOverlapInfo, ReadPbv,
ReadableFromFile, Sitemty, SitemtyFrameType, SubFrId, TimeBinnableType, TimeBinnableTypeAggregator, TimeBins,
WithLen,
pulse_offs_from_abs, ts_offs_from_abs, Appendable, FilterFittingInside, Fits, FitsInside, IsoDateTime,
RangeOverlapInfo, ReadPbv, ReadableFromFile, Sitemty, SitemtyFrameType, SubFrId, TimeBinnableType,
TimeBinnableTypeAggregator, TimeBins, WithLen,
};
use chrono::{TimeZone, Utc};
use err::Error;
@@ -451,6 +451,10 @@ pub struct WaveEventsCollectedResult<NTY> {
ts_off_ms: Vec<u64>,
#[serde(rename = "tsNs")]
ts_off_ns: Vec<u64>,
#[serde(rename = "pulseAnchor")]
pulse_anchor: u64,
#[serde(rename = "pulseOff")]
pulse_off: Vec<u64>,
values: Vec<Vec<NTY>>,
#[serde(skip_serializing_if = "crate::bool_is_false", rename = "finalisedRange")]
range_complete: bool,
@@ -502,10 +506,13 @@ where
fn result(self) -> Result<Self::Output, Error> {
let tst = ts_offs_from_abs(&self.vals.tss);
let (pulse_anchor, pulse_off) = pulse_offs_from_abs(&self.vals.pulses);
let ret = Self::Output {
ts_anchor_sec: tst.0,
ts_off_ms: tst.1,
ts_off_ns: tst.2,
pulse_anchor,
pulse_off,
values: self.vals.vals,
range_complete: self.range_complete,
timed_out: self.timed_out,

55
items/src/scalarevents.rs
View File

@@ -2,8 +2,8 @@ use crate::minmaxavgbins::MinMaxAvgBins;
use crate::numops::NumOps;
use crate::streams::{Collectable, Collector};
use crate::{
ts_offs_from_abs, Appendable, ByteEstimate, Clearable, EventAppendable, FilterFittingInside, Fits, FitsInside,
PushableIndex, RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, TimeBinnableType,
pulse_offs_from_abs, ts_offs_from_abs, Appendable, ByteEstimate, Clearable, EventAppendable, FilterFittingInside,
Fits, FitsInside, PushableIndex, RangeOverlapInfo, ReadPbv, ReadableFromFile, SitemtyFrameType, TimeBinnableType,
TimeBinnableTypeAggregator, WithLen, WithTimestamps,
};
use err::Error;
@@ -15,13 +15,40 @@ use tokio::fs::File;
// TODO in this module reduce clones.
// TODO add pulse.
#[derive(Serialize, Deserialize)]
pub struct ScalarEvents<NTY> {
pub tss: Vec<u64>,
pub pulses: Vec<u64>,
pub values: Vec<NTY>,
}
impl<NTY> ScalarEvents<NTY> {
#[inline(always)]
pub fn push(&mut self, ts: u64, pulse: u64, value: NTY) {
self.tss.push(ts);
self.pulses.push(pulse);
self.values.push(value);
}
// TODO should avoid the copies.
#[inline(always)]
pub fn extend_from_slice(&mut self, src: &Self)
where
NTY: Clone,
{
self.tss.extend_from_slice(&src.tss);
self.pulses.extend_from_slice(&src.pulses);
self.values.extend_from_slice(&src.values);
}
#[inline(always)]
pub fn clearx(&mut self) {
self.tss.clear();
self.pulses.clear();
self.values.clear();
}
}
impl<NTY> SitemtyFrameType for ScalarEvents<NTY>
where
NTY: NumOps,
@@ -33,6 +60,7 @@ impl<NTY> ScalarEvents<NTY> {
pub fn empty() -> Self {
Self {
tss: vec![],
pulses: vec![],
values: vec![],
}
}
@@ -148,8 +176,7 @@ where
NTY: NumOps,
{
fn push_index(&mut self, src: &Self, ix: usize) {
self.tss.push(src.tss[ix]);
self.values.push(src.values[ix].clone());
self.push(src.tss[ix], src.pulses[ix], src.values[ix].clone());
}
}
@@ -162,15 +189,13 @@ where
}
fn append(&mut self, src: &Self) {
self.tss.extend_from_slice(&src.tss);
self.values.extend_from_slice(&src.values);
self.extend_from_slice(src);
}
}
impl<NTY> Clearable for ScalarEvents<NTY> {
fn clear(&mut self) {
self.tss.clear();
self.values.clear();
ScalarEvents::<NTY>::clearx(self);
}
}
@@ -234,6 +259,10 @@ pub struct EventValuesCollectorOutput<NTY> {
ts_off_ms: Vec<u64>,
#[serde(rename = "tsNs")]
ts_off_ns: Vec<u64>,
#[serde(rename = "pulseAnchor")]
pulse_anchor: u64,
#[serde(rename = "pulseOff")]
pulse_off: Vec<u64>,
values: Vec<NTY>,
#[serde(skip_serializing_if = "crate::bool_is_false", rename = "finalisedRange")]
range_complete: bool,
@@ -262,10 +291,13 @@ where
fn result(self) -> Result<Self::Output, Error> {
let tst = ts_offs_from_abs(&self.vals.tss);
let (pulse_anchor, pulse_off) = pulse_offs_from_abs(&self.vals.pulses);
let ret = Self::Output {
ts_anchor_sec: tst.0,
ts_off_ms: tst.1,
ts_off_ns: tst.2,
pulse_anchor,
pulse_off,
values: self.vals.values,
range_complete: self.range_complete,
timed_out: self.timed_out,
@@ -501,10 +533,9 @@ where
{
type Value = NTY;
fn append_event(ret: Option<Self>, ts: u64, value: Self::Value) -> Self {
fn append_event(ret: Option<Self>, ts: u64, pulse: u64, value: Self::Value) -> Self {
let mut ret = if let Some(ret) = ret { ret } else { Self::empty() };
ret.tss.push(ts);
ret.values.push(value);
ret.push(ts, pulse, value);
ret
}
}

6
items/src/statsevents.rs
View File

@@ -392,13 +392,11 @@ impl TimeBinnableTypeAggregator for StatsEventsAggregator {
impl EventAppendable for StatsEvents {
type Value = f32;
fn append_event(ret: Option<Self>, ts: u64, _value: Self::Value) -> Self {
let mut ret = if let Some(ret) = ret { ret } else { Self::empty() };
ret.tss.push(ts);
fn append_event(ret: Option<Self>, _ts: u64, _pulse: u64, _value: Self::Value) -> Self {
let ret = if let Some(ret) = ret { ret } else { Self::empty() };
// TODO
error!("TODO statsevents append_event");
err::todo();
ret.pulses.push(42);
ret
}
}

18
items/src/waveevents.rs
View File

@@ -17,9 +17,18 @@ use tokio::fs::File;
#[derive(Debug, Serialize, Deserialize)]
pub struct WaveEvents<NTY> {
pub tss: Vec<u64>,
pub pulses: Vec<u64>,
pub vals: Vec<Vec<NTY>>,
}
impl<NTY> WaveEvents<NTY> {
pub fn push(&mut self, ts: u64, pulse: u64, value: Vec<NTY>) {
self.tss.push(ts);
self.pulses.push(pulse);
self.vals.push(value);
}
}
impl<NTY> WaveEvents<NTY> {
pub fn shape(&self) -> Result<Shape, Error> {
if let Some(k) = self.vals.first() {
@@ -42,6 +51,7 @@ impl<NTY> WaveEvents<NTY> {
pub fn empty() -> Self {
Self {
tss: vec![],
pulses: vec![],
vals: vec![],
}
}
@@ -319,10 +329,9 @@ where
{
type Value = Vec<NTY>;
fn append_event(ret: Option<Self>, ts: u64, value: Self::Value) -> Self {
fn append_event(ret: Option<Self>, ts: u64, pulse: u64, value: Self::Value) -> Self {
let mut ret = if let Some(ret) = ret { ret } else { Self::empty() };
ret.tss.push(ts);
ret.vals.push(value);
ret.push(ts, pulse, value);
ret
}
}
@@ -455,6 +464,7 @@ pub struct WavePlainProc<NTY> {
_m1: PhantomData<NTY>,
}
// TODO purpose?
impl<NTY> EventsNodeProcessor for WavePlainProc<NTY>
where
NTY: NumOps,
@@ -472,11 +482,13 @@ where
let n = if n > 5 { 5 } else { n };
WaveEvents {
tss: inp.tss,
pulses: inp.pulses,
vals: inp.vals.iter().map(|k| k[..n].to_vec()).collect(),
}
} else {
WaveEvents {
tss: inp.tss,
pulses: inp.pulses,
vals: inp.vals,
}
}