daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor and prepare to remove MinMaxAvgWaveBins

Browse Source
This commit is contained in:
Dominik Werder
2022-06-16 19:24:55 +02:00
parent 8090088448
commit f183f0bb28
6 changed files with 79 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dbconn/Cargo.toml
View File

@@ -30,3 +30,4 @@ err = { path = "../err" }
netpod = { path = "../netpod" } netpod = { path = "../netpod" }
parse = { path = "../parse" } parse = { path = "../parse" }
taskrun = { path = "../taskrun" } taskrun = { path = "../taskrun" }
items = { path = "../items" }

9
dbconn/src/bincache.rs Normal file
View File

@@ -0,0 +1,9 @@
use err::Error;
use scylla::Session as ScySession;
pub async fn search_channel_scylla<BINC>(_scy: &ScySession) -> Result<(), Error>
where
BINC: Clone,
{
todo!()
}

1
dbconn/src/dbconn.rs
View File

@@ -1,4 +1,5 @@
pub mod scan; pub mod scan;
pub mod bincache;
pub mod search; pub mod search;
pub mod pg { pub mod pg {
pub use tokio_postgres::{Client, Error}; pub use tokio_postgres::{Client, Error};

95
disk/src/binned/pbv.rs
View File

@@ -40,7 +40,8 @@ where
agg_kind: AggKind, agg_kind: AggKind,
node_config: NodeConfigCached, node_config: NodeConfigCached,
open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<File, io::Error>> + Send>>>, open_check_local_file: Option<Pin<Box<dyn Future<Output = Result<File, io::Error>> + Send>>>,
fut2: Option<Pin<Box<dyn Stream<Item = Sitemty<<ENP as EventsTypeAliases>::TimeBinOutput>> + Send>>>, stream_from_other_inputs:
Option<Pin<Box<dyn Stream<Item = Sitemty<<ENP as EventsTypeAliases>::TimeBinOutput>> + Send>>>,
read_from_cache: bool, read_from_cache: bool,
cache_written: bool, cache_written: bool,
data_complete: bool, data_complete: bool,
@@ -77,7 +78,7 @@ where
agg_kind, agg_kind,
node_config: node_config.clone(), node_config: node_config.clone(),
open_check_local_file: None, open_check_local_file: None,
fut2: None, stream_from_other_inputs: None,
read_from_cache: false, read_from_cache: false,
cache_written: false, cache_written: false,
data_complete: false, data_complete: false,
@@ -208,10 +209,10 @@ where
let range = self.query.patch().patch_range(); let range = self.query.patch().patch_range();
match PreBinnedPatchRange::covering_range(range, self.query.patch().bin_count() + 1) { match PreBinnedPatchRange::covering_range(range, self.query.patch().bin_count() + 1) {
Ok(Some(range)) => { Ok(Some(range)) => {
self.fut2 = Some(self.setup_from_higher_res_prebinned(range)?); self.stream_from_other_inputs = Some(self.setup_from_higher_res_prebinned(range)?);
} }
Ok(None) => { Ok(None) => {
self.fut2 = Some(self.setup_merged_from_remotes()?); self.stream_from_other_inputs = Some(self.setup_merged_from_remotes()?);
} }
Err(e) => return Err(e), Err(e) => return Err(e),
} }
@@ -223,6 +224,7 @@ where
mut fut: Pin<Box<dyn Future<Output = Result<WrittenPbCache, Error>> + Send>>, mut fut: Pin<Box<dyn Future<Output = Result<WrittenPbCache, Error>> + Send>>,
cx: &mut Context, cx: &mut Context,
) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> { ) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> {
trace!("poll_write_fut");
use Poll::*; use Poll::*;
match fut.poll_unpin(cx) { match fut.poll_unpin(cx) {
Ready(item) => { Ready(item) => {
@@ -268,6 +270,7 @@ where
>, >,
cx: &mut Context, cx: &mut Context,
) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> { ) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> {
trace!("poll_read_cache_fut");
use Poll::*; use Poll::*;
match fut.poll_unpin(cx) { match fut.poll_unpin(cx) {
Ready(item) => { Ready(item) => {
@@ -294,8 +297,10 @@ where
fn handle_data_complete( fn handle_data_complete(
self: &mut Self, self: &mut Self,
) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> { ) -> Poll<Option<Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>>> {
trace!("handle_data_complete");
use Poll::*; use Poll::*;
if self.cache_written { if self.cache_written {
// TODO can we ever get here?
if self.range_complete_observed { if self.range_complete_observed {
self.range_complete_emitted = true; self.range_complete_emitted = true;
let item = RangeCompletableItem::RangeComplete; let item = RangeCompletableItem::RangeComplete;
@@ -305,6 +310,7 @@ where
Ready(None) Ready(None)
} }
} else if self.read_from_cache { } else if self.read_from_cache {
// TODO refactor: raising cache_written even though we did not actually write is misleading.
self.cache_written = true; self.cache_written = true;
self.all_done = true; self.all_done = true;
Ready(None) Ready(None)
@@ -333,6 +339,7 @@ where
} }
} }
_ => { _ => {
// TODO refactor: raising cache_written even though we did not actually write is misleading.
self.cache_written = true; self.cache_written = true;
self.all_done = true; self.all_done = true;
Ready(None) Ready(None)
@@ -341,7 +348,7 @@ where
} }
} }
fn poll_fut2( fn poll_stream_from_other_inputs(
self: &mut Self, self: &mut Self,
mut fut: Pin< mut fut: Pin<
Box< Box<
@@ -358,26 +365,29 @@ where
use Poll::*; use Poll::*;
match fut.poll_next_unpin(cx) { match fut.poll_next_unpin(cx) {
Ready(Some(k)) => match k { Ready(Some(k)) => match k {
Ok(item) => match item { Ok(item) => {
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))), self.stream_from_other_inputs = Some(fut);
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))), match item {
StreamItem::DataItem(item) => match item { StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
RangeCompletableItem::RangeComplete => { StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
self.range_complete_observed = true; StreamItem::DataItem(item) => match item {
Ready(None) RangeCompletableItem::RangeComplete => {
} self.range_complete_observed = true;
RangeCompletableItem::Data(item) => { Ready(None)
if let Some(values) = &mut self.values {
values.append(&item);
} else {
let mut values = item.empty_like_self();
values.append(&item);
self.values = Some(values);
} }
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(item))))) RangeCompletableItem::Data(item) => {
} if let Some(values) = &mut self.values {
}, values.append(&item);
}, } else {
let mut values = item.empty_like_self();
values.append(&item);
self.values = Some(values);
}
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))))
}
},
}
}
Err(e) => { Err(e) => {
self.errored = true; self.errored = true;
Ready(Some(Err(e))) Ready(Some(Err(e)))
@@ -388,7 +398,7 @@ where
Ready(None) Ready(None)
} }
Pending => { Pending => {
self.fut2 = Some(fut); self.stream_from_other_inputs = Some(fut);
Pending Pending
} }
} }
@@ -416,7 +426,7 @@ where
// TODO other error kinds // TODO other error kinds
io::ErrorKind::NotFound => match self.try_setup_fetch_prebinned_higher_res() { io::ErrorKind::NotFound => match self.try_setup_fetch_prebinned_higher_res() {
Ok(_) => { Ok(_) => {
if self.fut2.is_none() { if self.stream_from_other_inputs.is_none() {
let e = let e =
Err(Error::with_msg(format!("try_setup_fetch_prebinned_higher_res failed"))); Err(Error::with_msg(format!("try_setup_fetch_prebinned_higher_res failed")));
self.errored = true; self.errored = true;
@@ -447,10 +457,16 @@ where
} }
} }
} }
}
fn _check_for_existing_cached_data(&mut self) -> Result<(), Error> { macro_rules! some_or_continue {
todo!() ($x:expr) => {
} if let Ready(None) = $x {
continue;
} else {
$x
}
};
} }
impl<NTY, END, EVS, ENP> Stream for PreBinnedValueStream<NTY, END, EVS, ENP> impl<NTY, END, EVS, ENP> Stream for PreBinnedValueStream<NTY, END, EVS, ENP>
@@ -468,7 +484,7 @@ where
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*; use Poll::*;
'outer: loop { loop {
break if self.completed { break if self.completed {
panic!("PreBinnedValueStream poll_next on completed"); panic!("PreBinnedValueStream poll_next on completed");
} else if self.errored { } else if self.errored {
@@ -480,18 +496,23 @@ where
} else if let Some(item) = self.streamlog.pop() { } else if let Some(item) = self.streamlog.pop() {
Ready(Some(Ok(StreamItem::Log(item)))) Ready(Some(Ok(StreamItem::Log(item))))
} else if let Some(fut) = self.write_fut.take() { } else if let Some(fut) = self.write_fut.take() {
Self::poll_write_fut(&mut self, fut, cx) let x = Self::poll_write_fut(&mut self, fut, cx);
some_or_continue!(x)
} else if let Some(fut) = self.read_cache_fut.take() { } else if let Some(fut) = self.read_cache_fut.take() {
Self::poll_read_cache_fut(&mut self, fut, cx) let x = Self::poll_read_cache_fut(&mut self, fut, cx);
some_or_continue!(x)
} else if self.range_complete_emitted { } else if self.range_complete_emitted {
self.completed = true; self.completed = true;
Ready(None) Ready(None)
} else if self.data_complete { } else if self.data_complete {
Self::handle_data_complete(&mut self) let x = Self::handle_data_complete(&mut self);
} else if let Some(fut) = self.fut2.take() { some_or_continue!(x)
Self::poll_fut2(&mut self, fut, cx) } else if let Some(fut) = self.stream_from_other_inputs.take() {
let x = Self::poll_stream_from_other_inputs(&mut self, fut, cx);
some_or_continue!(x)
} else if let Some(fut) = self.open_check_local_file.take() { } else if let Some(fut) = self.open_check_local_file.take() {
Self::poll_open_check_local_file(&mut self, fut, cx) let x = Self::poll_open_check_local_file(&mut self, fut, cx);
some_or_continue!(x)
} else { } else {
let cfd = CacheFileDesc::new( let cfd = CacheFileDesc::new(
self.query.channel().clone(), self.query.channel().clone(),
@@ -504,7 +525,7 @@ where
}; };
let fut = async { OpenOptions::new().read(true).open(path).await }; let fut = async { OpenOptions::new().read(true).open(path).await };
self.open_check_local_file = Some(Box::pin(fut)); self.open_check_local_file = Some(Box::pin(fut));
continue 'outer; continue;
}; };
} }
} }

19
items/src/xbinnedwaveevents.rs
View File

@@ -1,6 +1,4 @@
use std::mem; use crate::minmaxavgdim1bins::MinMaxAvgDim1Bins;
use crate::minmaxavgwavebins::MinMaxAvgWaveBins;
use crate::numops::NumOps; use crate::numops::NumOps;
use crate::streams::{Collectable, Collector}; use crate::streams::{Collectable, Collector};
use crate::{ use crate::{
@@ -10,9 +8,10 @@ use crate::{
}; };
use err::Error; use err::Error;
use netpod::log::*; use netpod::log::*;
use netpod::timeunits::{MS, SEC}; use netpod::timeunits::*;
use netpod::NanoRange; use netpod::NanoRange;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::mem;
use tokio::fs::File; use tokio::fs::File;
// TODO rename Wave -> Dim1 // TODO rename Wave -> Dim1
@@ -177,7 +176,7 @@ impl<NTY> TimeBinnableType for XBinnedWaveEvents<NTY>
where where
NTY: NumOps, NTY: NumOps,
{ {
type Output = MinMaxAvgWaveBins<NTY>; type Output = MinMaxAvgDim1Bins<NTY>;
type Aggregator = XBinnedWaveEventsAggregator<NTY>; type Aggregator = XBinnedWaveEventsAggregator<NTY>;
fn aggregator(range: NanoRange, x_bin_count: usize, do_time_weight: bool) -> Self::Aggregator { fn aggregator(range: NanoRange, x_bin_count: usize, do_time_weight: bool) -> Self::Aggregator {
@@ -324,7 +323,7 @@ where
} }
} }
fn result_reset_unweight(&mut self, range: NanoRange, _expand: bool) -> MinMaxAvgWaveBins<NTY> { fn result_reset_unweight(&mut self, range: NanoRange, _expand: bool) -> MinMaxAvgDim1Bins<NTY> {
let avg = if self.sumc == 0 { let avg = if self.sumc == 0 {
None None
} else { } else {
@@ -332,7 +331,7 @@ where
}; };
let min = mem::replace(&mut self.min, None); let min = mem::replace(&mut self.min, None);
let max = mem::replace(&mut self.max, None); let max = mem::replace(&mut self.max, None);
let ret = MinMaxAvgWaveBins { let ret = MinMaxAvgDim1Bins {
ts1s: vec![self.range.beg], ts1s: vec![self.range.beg],
ts2s: vec![self.range.end], ts2s: vec![self.range.end],
counts: vec![self.count], counts: vec![self.count],
@@ -350,7 +349,7 @@ where
ret ret
} }
fn result_reset_time_weight(&mut self, range: NanoRange, expand: bool) -> MinMaxAvgWaveBins<NTY> { fn result_reset_time_weight(&mut self, range: NanoRange, expand: bool) -> MinMaxAvgDim1Bins<NTY> {
// TODO check callsite for correct expand status. // TODO check callsite for correct expand status.
if true || expand { if true || expand {
self.apply_event_time_weight(self.range.end); self.apply_event_time_weight(self.range.end);
@@ -363,7 +362,7 @@ where
}; };
let min = mem::replace(&mut self.min, None); let min = mem::replace(&mut self.min, None);
let max = mem::replace(&mut self.max, None); let max = mem::replace(&mut self.max, None);
let ret = MinMaxAvgWaveBins { let ret = MinMaxAvgDim1Bins {
ts1s: vec![self.range.beg], ts1s: vec![self.range.beg],
ts2s: vec![self.range.end], ts2s: vec![self.range.end],
counts: vec![self.count], counts: vec![self.count],
@@ -387,7 +386,7 @@ where
NTY: NumOps, NTY: NumOps,
{ {
type Input = XBinnedWaveEvents<NTY>; type Input = XBinnedWaveEvents<NTY>;
type Output = MinMaxAvgWaveBins<NTY>; type Output = MinMaxAvgDim1Bins<NTY>;
fn range(&self) -> &NanoRange { fn range(&self) -> &NanoRange {
&self.range &self.range

10
taskrun/src/taskrun.rs
View File

@@ -104,15 +104,7 @@ pub fn tracing_init() {
//"tokio=trace", //"tokio=trace",
//"runtime=trace", //"runtime=trace",
"info", "info",
"archapp::archeng=info", "disk::binned::pbv=trace",
"archapp::archeng::datablockstream=info",
"archapp::archeng::indextree=info",
"archapp::archeng::blockrefstream=info",
"archapp::archeng::blockstream=info",
"archapp::archeng::ringbuf=info",
"archapp::archeng::backreadbuf=info",
"archapp::archeng::pipe=debug",
"archapp::storagemerge=info",
"[log_span_d]=debug", "[log_span_d]=debug",
"[log_span_t]=trace", "[log_span_t]=trace",
] ]