daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2023-02-10 11:37:17 +01:00
parent 4694f98758
commit 4d9a33b77f
25 changed files with 609 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
items_2/src/channelevents.rs
View File

@@ -504,28 +504,6 @@ impl crate::merger::Mergeable for ChannelEvents {
}
}
fn is_compatible_target(&self, tgt: &Self) -> bool {
use ChannelEvents::*;
match self {
Events(_) => {
// TODO better to delegate this to inner type?
if let Events(_) = tgt {
true
} else {
false
}
}
Status(_) => {
// TODO better to delegate this to inner type?
if let Status(_) = tgt {
true
} else {
false
}
}
}
}
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
match self {
ChannelEvents::Events(k) => ChannelEvents::Events(k.move_into_fresh(ts_end)),
@@ -545,6 +523,69 @@ impl crate::merger::Mergeable for ChannelEvents {
},
}
}
fn new_empty(&self) -> Self {
match self {
ChannelEvents::Events(k) => ChannelEvents::Events(k.new_empty()),
ChannelEvents::Status(k) => ChannelEvents::Status(k.clone()),
}
}
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), merger::MergeError> {
match self {
ChannelEvents::Events(k) => match dst {
ChannelEvents::Events(j) => k.drain_into(j, range),
ChannelEvents::Status(_) => Err(merger::MergeError::NotCompatible),
},
ChannelEvents::Status(k) => match dst {
ChannelEvents::Events(_) => Err(merger::MergeError::NotCompatible),
ChannelEvents::Status(j) => {
// TODO must have some empty-value for the status container.
*j = k.clone();
Ok(())
}
},
}
}
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize> {
match self {
ChannelEvents::Events(k) => k.find_lowest_index_gt(ts),
ChannelEvents::Status(k) => {
if k.ts > ts {
Some(0)
} else {
None
}
}
}
}
fn find_lowest_index_ge(&self, ts: u64) -> Option<usize> {
match self {
ChannelEvents::Events(k) => k.find_lowest_index_ge(ts),
ChannelEvents::Status(k) => {
if k.ts >= ts {
Some(0)
} else {
None
}
}
}
}
fn find_highest_index_lt(&self, ts: u64) -> Option<usize> {
match self {
ChannelEvents::Events(k) => k.find_highest_index_lt(ts),
ChannelEvents::Status(k) => {
if k.ts < ts {
Some(0)
} else {
None
}
}
}
}
}
impl Collectable for ChannelEvents {

47
items_2/src/eventsdim0.rs
View File

@@ -382,7 +382,6 @@ impl<NTY: ScalarOps> items_0::collect_s::CollectorType for EventsDim0Collector<N
if !ret.is_valid() {
error!("invalid:\n{}", ret.info_str());
}
info!("EventsDim0CollectorOutput {ret:?}");
Ok(ret)
}
}
@@ -805,6 +804,52 @@ impl<NTY: ScalarOps> Events for EventsDim0<NTY> {
}
}
fn new_empty(&self) -> Box<dyn Events> {
Box::new(Self::empty())
}
fn drain_into(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrelated traits. Simplify.
if let Some(dst) = dst.as_mut().as_any_mut().downcast_mut::<Self>() {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;
dst.tss.extend(self.tss.drain(r.clone()));
dst.pulses.extend(self.pulses.drain(r.clone()));
dst.values.extend(self.values.drain(r.clone()));
Ok(())
} else {
error!("downcast to EventsDim0 FAILED");
Err(())
}
}
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate() {
if m > ts {
return Some(i);
}
}
None
}
fn find_lowest_index_ge(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate() {
if m >= ts {
return Some(i);
}
}
None
}
fn find_highest_index_lt(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate().rev() {
if m < ts {
return Some(i);
}
}
None
}
fn ts_min(&self) -> Option<u64> {
self.tss.front().map(|&x| x)
}

46
items_2/src/eventsdim1.rs
View File

@@ -756,6 +756,52 @@ impl<NTY: ScalarOps> Events for EventsDim1<NTY> {
}
}
fn new_empty(&self) -> Box<dyn Events> {
Box::new(Self::empty())
}
fn drain_into(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), ()> {
// TODO as_any and as_any_mut are declared on unrelated traits. Simplify.
if let Some(dst) = dst.as_mut().as_any_mut().downcast_mut::<Self>() {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;
dst.tss.extend(self.tss.drain(r.clone()));
dst.pulses.extend(self.pulses.drain(r.clone()));
dst.values.extend(self.values.drain(r.clone()));
Ok(())
} else {
error!("downcast to EventsDim0 FAILED");
Err(())
}
}
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate() {
if m > ts {
return Some(i);
}
}
None
}
fn find_lowest_index_ge(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate() {
if m >= ts {
return Some(i);
}
}
None
}
fn find_highest_index_lt(&self, ts: u64) -> Option<usize> {
for (i, &m) in self.tss.iter().enumerate().rev() {
if m < ts {
return Some(i);
}
}
None
}
fn ts_min(&self) -> Option<u64> {
self.tss.front().map(|&x| x)
}

33
items_2/src/items_2.rs
View File

@@ -51,7 +51,6 @@ pub fn bool_is_false(x: &bool) -> bool {
pub fn ts_offs_from_abs(tss: &[u64]) -> (u64, VecDeque<u64>, VecDeque<u64>) {
let ts_anchor_sec = tss.first().map_or(0, |&k| k) / SEC;
info!("ts_offs_from_abs ts_anchor_sec {ts_anchor_sec}");
let ts_anchor_ns = ts_anchor_sec * SEC;
let ts_off_ms: VecDeque<_> = tss.iter().map(|&k| (k - ts_anchor_ns) / MS).collect();
let ts_off_ns = tss
@@ -74,12 +73,7 @@ pub fn ts_offs_from_abs_with_anchor(ts_anchor_sec: u64, tss: &[u64]) -> (VecDequ
}
pub fn pulse_offs_from_abs(pulse: &[u64]) -> (u64, VecDeque<u64>) {
error!("pulse_offs_from_abs {} DATA", pulse.len());
for x in pulse {
error!("{x}");
}
let pulse_anchor = pulse.first().map_or(0, |&k| k) / 10000 * 10000;
info!("pulse_offs_from_abs pulse_anchor {pulse_anchor}");
let pulse_off = pulse.iter().map(|&k| k - pulse_anchor).collect();
(pulse_anchor, pulse_off)
}
@@ -188,11 +182,6 @@ impl crate::merger::Mergeable for Box<dyn Events> {
self.as_ref().ts_max()
}
fn is_compatible_target(&self, _tgt: &Self) -> bool {
// TODO currently unused
todo!()
}
fn move_into_fresh(&mut self, ts_end: u64) -> Self {
self.as_mut().move_into_fresh(ts_end)
}
@@ -202,6 +191,28 @@ impl crate::merger::Mergeable for Box<dyn Events> {
.move_into_existing(tgt, ts_end)
.map_err(|()| merger::MergeError::NotCompatible)
}
fn new_empty(&self) -> Self {
self.as_ref().new_empty()
}
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), merger::MergeError> {
self.as_mut()
.drain_into(dst, range)
.map_err(|()| merger::MergeError::NotCompatible)
}
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize> {
self.as_ref().find_lowest_index_gt(ts)
}
fn find_lowest_index_ge(&self, ts: u64) -> Option<usize> {
self.as_ref().find_lowest_index_ge(ts)
}
fn find_highest_index_lt(&self, ts: u64) -> Option<usize> {
self.as_ref().find_highest_index_lt(ts)
}
}
// TODO rename to `Typed`

42
items_2/src/merger.rs
View File

@@ -5,7 +5,7 @@ use items::{RangeCompletableItem, Sitemty, StreamItem};
use netpod::log::*;
use std::collections::VecDeque;
use std::fmt;
use std::ops::ControlFlow;
use std::ops::{ControlFlow, RangeBounds};
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -43,12 +43,17 @@ pub trait Mergeable<Rhs = Self>: fmt::Debug + Unpin {
fn len(&self) -> usize;
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
// TODO remove, useless.
fn is_compatible_target(&self, tgt: &Rhs) -> bool;
// TODO rename to `append_*` to make it clear that they simply append, but not re-sort.
// TODO remove, superseded.
fn move_into_fresh(&mut self, ts_end: u64) -> Rhs;
fn move_into_existing(&mut self, tgt: &mut Rhs, ts_end: u64) -> Result<(), MergeError>;
// TODO: split the logic into: make fresh container, and a single drain_into method. Or is there any advantage in having both?
fn new_empty(&self) -> Self;
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError>;
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>;
fn find_lowest_index_ge(&self, ts: u64) -> Option<usize>;
fn find_highest_index_lt(&self, ts: u64) -> Option<usize>;
}
type MergeInp<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
@@ -107,6 +112,19 @@ where
}
}
fn drain_into_upto(src: &mut T, dst: &mut T, upto: u64) -> Result<(), MergeError> {
match src.find_lowest_index_gt(upto) {
Some(ilgt) => {
src.drain_into(dst, (0, ilgt))?;
}
None => {
// TODO should not be here.
src.drain_into(dst, (0, src.len()))?;
}
}
Ok(())
}
fn take_into_output_all(&mut self, src: &mut T) -> Result<(), MergeError> {
// TODO optimize the case when some large batch should be added to some existing small batch already in out.
// TODO maybe use two output slots?
@@ -116,14 +134,15 @@ where
fn take_into_output_upto(&mut self, src: &mut T, upto: u64) -> Result<(), MergeError> {
// TODO optimize the case when some large batch should be added to some existing small batch already in out.
// TODO maybe use two output slots?
if self.out.is_none() {
trace2!("move into fresh");
self.out = Some(src.move_into_fresh(upto));
Ok(())
if let Some(out) = self.out.as_mut() {
Self::drain_into_upto(src, out, upto)?;
} else {
let out = self.out.as_mut().unwrap();
src.move_into_existing(out, upto)
trace2!("move into fresh");
let mut fresh = src.new_empty();
Self::drain_into_upto(src, &mut fresh, upto)?;
self.out = Some(fresh);
}
Ok(())
}
fn process(mut self: Pin<&mut Self>, _cx: &mut Context) -> Result<ControlFlow<()>, Error> {
@@ -159,7 +178,7 @@ where
}
}
}
info!("tslows {tslows:?}");
trace4!("tslows {tslows:?}");
if let Some((il0, _tl0)) = tslows[0] {
if let Some((_il1, tl1)) = tslows[1] {
// There is a second input, take only up to the second highest timestamp
@@ -167,6 +186,7 @@ where
if let Some(th0) = item.ts_max() {
if th0 <= tl1 {
// Can take the whole item
// TODO gather stats about this case. Should be never for databuffer, and often for scylla.
let mut item = self.items[il0].take().unwrap();
trace3!("Take all from item {item:?}");
match self.take_into_output_all(&mut item) {