daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce db connections, improve merge mt/lt

Browse Source
This commit is contained in:
Dominik Werder
2024-06-19 11:20:28 +02:00
parent 3a77d116f6
commit 6b4fa3f7e1
29 changed files with 669 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
crates/items_2/src/channelevents.rs
View File

@@ -154,6 +154,15 @@ pub enum ChannelEvents {
Status(Option<ConnStatusEvent>),
}
impl ChannelEvents {
pub fn is_events(&self) -> bool {
match self {
ChannelEvents::Events(_) => true,
ChannelEvents::Status(_) => false,
}
}
}
impl TypeName for ChannelEvents {
fn type_name(&self) -> String {
any::type_name::<Self>().into()
@@ -702,6 +711,17 @@ impl Mergeable for ChannelEvents {
}
}
fn clear(&mut self) {
match self {
ChannelEvents::Events(x) => {
Mergeable::clear(x);
}
ChannelEvents::Status(x) => {
*x = None;
}
}
}
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError> {
match self {
ChannelEvents::Events(k) => match dst {
@@ -829,7 +849,10 @@ impl Events for ChannelEvents {
}
fn verify(&self) -> bool {
todo!()
match self {
ChannelEvents::Events(x) => Events::verify(x),
ChannelEvents::Status(_) => panic!(),
}
}
fn output_info(&self) -> String {
@@ -861,11 +884,26 @@ impl Events for ChannelEvents {
}
fn new_empty_evs(&self) -> Box<dyn Events> {
todo!()
match self {
ChannelEvents::Events(x) => Events::new_empty_evs(x),
ChannelEvents::Status(_) => panic!(),
}
}
fn drain_into_evs(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), MergeError> {
todo!()
fn drain_into_evs(&mut self, dst: &mut dyn Events, range: (usize, usize)) -> Result<(), MergeError> {
let dst2 = if let Some(x) = dst.as_any_mut().downcast_mut::<Self>() {
// debug!("unwrapped dst ChannelEvents as well");
x
} else {
panic!("dst is not ChannelEvents");
};
match self {
ChannelEvents::Events(k) => match dst2 {
ChannelEvents::Events(j) => Events::drain_into_evs(k, j, range),
ChannelEvents::Status(_) => panic!("dst is not events"),
},
ChannelEvents::Status(_) => panic!("self is not events"),
}
}
fn find_lowest_index_gt_evs(&self, ts: u64) -> Option<usize> {
@@ -896,11 +934,14 @@ impl Events for ChannelEvents {
todo!()
}
fn tss(&self) -> &std::collections::VecDeque<u64> {
todo!()
fn tss(&self) -> &VecDeque<u64> {
match self {
ChannelEvents::Events(x) => Events::tss(x),
ChannelEvents::Status(_) => panic!(),
}
}
fn pulses(&self) -> &std::collections::VecDeque<u64> {
fn pulses(&self) -> &VecDeque<u64> {
todo!()
}
@@ -934,6 +975,15 @@ impl Events for ChannelEvents {
}
}
}
fn clear(&mut self) {
match self {
ChannelEvents::Events(x) => Events::clear(x.as_mut()),
ChannelEvents::Status(x) => {
*x = None;
}
}
}
}
impl Collectable for ChannelEvents {

11
crates/items_2/src/eventfull.rs
View File

@@ -201,6 +201,17 @@ impl Mergeable for EventFull {
Empty::empty()
}
fn clear(&mut self) {
self.tss.clear();
self.pulses.clear();
self.blobs.clear();
self.scalar_types.clear();
self.be.clear();
self.shapes.clear();
self.comps.clear();
self.entry_payload_max = 0;
}
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError> {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;

17
crates/items_2/src/eventsdim0.rs
View File

@@ -859,9 +859,9 @@ impl<STY: ScalarOps> Events for EventsDim0<STY> {
Box::new(Self::empty())
}
fn drain_into_evs(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), MergeError> {
fn drain_into_evs(&mut self, dst: &mut dyn Events, range: (usize, usize)) -> Result<(), MergeError> {
// TODO as_any and as_any_mut are declared on unrelated traits. Simplify.
if let Some(dst) = dst.as_mut().as_any_mut().downcast_mut::<Self>() {
if let Some(dst) = dst.as_any_mut().downcast_mut::<Self>() {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;
dst.tss.extend(self.tss.drain(r.clone()));
@@ -869,7 +869,12 @@ impl<STY: ScalarOps> Events for EventsDim0<STY> {
dst.values.extend(self.values.drain(r.clone()));
Ok(())
} else {
error!("downcast to EventsDim0 FAILED");
error!(
"downcast to EventsDim0 FAILED\n\n{}\n\n{}\n\n",
self.type_name(),
dst.type_name()
);
panic!();
Err(MergeError::NotCompatible)
}
}
@@ -989,6 +994,12 @@ impl<STY: ScalarOps> Events for EventsDim0<STY> {
ciborium::into_writer(&ret, &mut buf).unwrap();
buf
}
fn clear(&mut self) {
self.tss.clear();
self.pulses.clear();
self.values.clear();
}
}
#[derive(Debug)]

10
crates/items_2/src/eventsdim1.rs
View File

@@ -813,9 +813,9 @@ impl<STY: ScalarOps> Events for EventsDim1<STY> {
Box::new(Self::empty())
}
fn drain_into_evs(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), MergeError> {
fn drain_into_evs(&mut self, dst: &mut dyn Events, range: (usize, usize)) -> Result<(), MergeError> {
// TODO as_any and as_any_mut are declared on unrelated traits. Simplify.
if let Some(dst) = dst.as_mut().as_any_mut().downcast_mut::<Self>() {
if let Some(dst) = dst.as_any_mut().downcast_mut::<Self>() {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;
dst.tss.extend(self.tss.drain(r.clone()));
@@ -949,6 +949,12 @@ impl<STY: ScalarOps> Events for EventsDim1<STY> {
ciborium::into_writer(&ret, &mut buf).unwrap();
buf
}
fn clear(&mut self) {
self.tss.clear();
self.pulses.clear();
self.values.clear();
}
}
#[derive(Debug)]

12
crates/items_2/src/eventsxbindim0.rs
View File

@@ -261,9 +261,9 @@ impl<STY: ScalarOps> Events for EventsXbinDim0<STY> {
Box::new(Self::empty())
}
fn drain_into_evs(&mut self, dst: &mut Box<dyn Events>, range: (usize, usize)) -> Result<(), MergeError> {
fn drain_into_evs(&mut self, dst: &mut dyn Events, range: (usize, usize)) -> Result<(), MergeError> {
// TODO as_any and as_any_mut are declared on unrelated traits. Simplify.
if let Some(dst) = dst.as_mut().as_any_mut().downcast_mut::<Self>() {
if let Some(dst) = dst.as_any_mut().downcast_mut::<Self>() {
// TODO make it harder to forget new members when the struct may get modified in the future
let r = range.0..range.1;
dst.tss.extend(self.tss.drain(r.clone()));
@@ -365,6 +365,14 @@ impl<STY: ScalarOps> Events for EventsXbinDim0<STY> {
fn to_cbor_vec_u8(&self) -> Vec<u8> {
todo!()
}
fn clear(&mut self) {
self.tss.clear();
self.pulses.clear();
self.mins.clear();
self.maxs.clear();
self.avgs.clear();
}
}
#[derive(Debug)]

4
crates/items_2/src/items_2.rs
View File

@@ -170,6 +170,10 @@ impl Mergeable for Box<dyn Events> {
self.as_ref().new_empty_evs()
}
fn clear(&mut self) {
Events::clear(self.as_mut())
}
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError> {
self.as_mut().drain_into_evs(dst, range)
}

1
crates/items_2/src/merger.rs
View File

@@ -47,6 +47,7 @@ pub trait Mergeable<Rhs = Self>: fmt::Debug + WithLen + ByteEstimate + Unpin {
fn ts_min(&self) -> Option<u64>;
fn ts_max(&self) -> Option<u64>;
fn new_empty(&self) -> Self;
fn clear(&mut self);
// TODO when MergeError::Full gets returned, any guarantees about what has been modified or kept unchanged?
fn drain_into(&mut self, dst: &mut Self, range: (usize, usize)) -> Result<(), MergeError>;
fn find_lowest_index_gt(&self, ts: u64) -> Option<usize>;