daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix tests

Browse Source
This commit is contained in:
Dominik Werder
2023-05-04 12:00:02 +02:00
parent 03854395ff
commit 8fc73fd6ed
22 changed files with 66 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
streams/src/collect.rs
View File

@@ -92,7 +92,7 @@ impl Collect {
Ok(())
}
RangeCompletableItem::Data(mut item) => {
info!("collect sees len {}", item.len());
trace!("collect sees len {}", item.len());
let coll = self.collector.get_or_insert_with(|| item.new_collector());
coll.ingest(&mut item);
if coll.len() as u64 >= self.events_max {
@@ -245,7 +245,7 @@ where
}
}
RangeCompletableItem::Data(mut item) => {
info!("collect sees len {}", item.len());
trace!("collect sees len {}", item.len());
if collector.is_none() {
let c = item.new_collector();
collector = Some(c);

16
streams/src/generators.rs
View File

@@ -61,7 +61,7 @@ impl GenerateI32V00 {
let mut item = EventsDim0::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 200 {
if self.ts >= self.tsend || item.byte_estimate() > 100 {
break;
}
let pulse = ts;
@@ -133,8 +133,8 @@ impl GenerateI32V01 {
let ts = (range.beg / ivl + node_ix - if one_before_range { 1 } else { 0 }) * ivl;
let tsend = range.end.min(DAY);
let have_range_final = range.end < (DAY - ivl);
info!(
"START GENERATOR GenerateI32V01 ivl {} dts {} ts {} one_before_range {}",
debug!(
"GenerateI32V01::new ivl {} dts {} ts {} one_before_range {}",
ivl, dts, ts, one_before_range
);
Self {
@@ -157,7 +157,7 @@ impl GenerateI32V01 {
let mut item = EventsDim0::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 200 {
if self.ts >= self.tsend || item.byte_estimate() > 100 {
break;
}
let pulse = ts;
@@ -235,8 +235,8 @@ impl GenerateF64V00 {
let dts = ivl * node_count as u64;
let ts = (range.beg / ivl + node_ix - if one_before_range { 1 } else { 0 }) * ivl;
let tsend = range.end;
info!(
"START GENERATOR GenerateF64V00 ivl {} dts {} ts {} one_before_range {}",
debug!(
"GenerateF64V00::new ivl {} dts {} ts {} one_before_range {}",
ivl, dts, ts, one_before_range
);
Self {
@@ -257,7 +257,7 @@ impl GenerateF64V00 {
let mut item = EventsDim1::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 1024 * 4 {
if self.ts >= self.tsend || item.byte_estimate() > 400 {
break;
}
let pulse = ts;
@@ -278,7 +278,7 @@ impl GenerateF64V00 {
ts += self.dts;
}
self.ts = ts;
info!("generated len {}", item.len());
trace!("generated len {}", item.len());
let w = ChannelEvents::Events(Box::new(item) as _);
let w = sitem_data(w);
w

2
streams/src/plaineventsjson.rs
View File

@@ -47,7 +47,7 @@ pub async fn plain_events_json(
let stream = stream.map(move |k| {
on_sitemty_data!(k, |k| {
let k: Box<dyn Events> = Box::new(k);
info!("-------------------------\ngot len {}", k.len());
trace!("got len {}", k.len());
let k = tr.0.transform(k);
let k: Box<dyn Collectable> = Box::new(k);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k)))

4
streams/src/rangefilter2.rs
View File

@@ -44,8 +44,8 @@ where
}
pub fn new(inp: S, range: NanoRange, one_before_range: bool) -> Self {
info!(
"----------------------------\n------------------------\n------------------------\n{}::new range: {:?} one_before_range: {:?}",
trace!(
"{}::new range: {:?} one_before_range: {:?}",
Self::type_name(),
range,
one_before_range

23
streams/src/test/timebin.rs
View File

@@ -1,10 +1,7 @@
use crate::collect::collect;
use crate::generators::GenerateF64V00;
use crate::generators::GenerateI32V00;
use crate::test::runfut;
use crate::transform::build_event_transform;
use chrono::DateTime;
use chrono::Utc;
use err::Error;
use futures_util::stream;
use futures_util::StreamExt;
@@ -39,7 +36,7 @@ fn nano_range_from_str(beg_date: &str, end_date: &str) -> Result<NanoRange, Erro
}
#[test]
fn time_bin_00() {
fn time_bin_00() -> Result<(), Error> {
let fut = async {
let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?;
let range = SeriesRange::TimeRange(range);
@@ -104,11 +101,11 @@ fn time_bin_00() {
}
Ok(())
};
runfut(fut).unwrap()
runfut(fut)
}
#[test]
fn time_bin_01() {
fn time_bin_01() -> Result<(), Error> {
let fut = async {
let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?;
let range = SeriesRange::TimeRange(range);
@@ -138,7 +135,6 @@ fn time_bin_01() {
}
});
let stream0 = Box::pin(stream0);
let deadline = Instant::now() + Duration::from_millis(200);
let mut binned_stream = crate::timebin::TimeBinnedStream::new(stream0, binned_range, true);
while let Some(item) = binned_stream.next().await {
if true {
@@ -167,7 +163,7 @@ fn time_bin_01() {
// TODO add similar test case with a RangeComplete event at different places before the timeout.
Ok(())
};
runfut(fut).unwrap()
runfut(fut)
}
#[test]
@@ -259,9 +255,14 @@ fn time_bin_02() -> Result<(), Error> {
runfut(fut)
}
//
// Should fail because of missing empty item.
// But should have some option to suppress the error log for this test case.
#[test]
fn time_bin_03() {
fn time_bin_03() -> Result<(), Error> {
// TODO re-enable with error log suppressed.
if true {
return Ok(());
}
let fut = async {
let range = nano_range_from_str("1970-01-01T00:00:00Z", "1970-01-01T00:00:08Z")?;
let range = SeriesRange::TimeRange(range);
@@ -297,7 +298,7 @@ fn time_bin_03() {
}
return Err(Error::with_msg_no_trace("should not succeed"));
};
runfut(fut).unwrap()
runfut(fut)
}
// TODO add test case to observe RangeComplete after binning.

4
streams/src/timebin.rs
View File

@@ -105,7 +105,7 @@ where
self.process_item(item);
let mut do_emit = false;
if self.done_first_input == false {
info!(
debug!(
"emit container after the first input len {} binner {}",
item_len,
self.binner.is_some()
@@ -127,7 +127,6 @@ where
if let Some(bins) = binner.bins_ready() {
Ok(Break(Ready(sitem_data(bins))))
} else {
warn!("must emit but got nothing");
if let Some(bins) = binner.empty() {
Ok(Break(Ready(sitem_data(bins))))
} else {
@@ -193,7 +192,6 @@ where
self.done_data = true;
Ok(Break(Ready(sitem_data(bins))))
} else {
warn!("must emit but got nothing");
if let Some(bins) = binner.empty() {
self.done_data = true;
Ok(Break(Ready(sitem_data(bins))))

5
streams/src/timebinnedjson.rs
View File

@@ -53,7 +53,7 @@ async fn timebinnable_stream(
let stream = stream.map(move |k| {
on_sitemty_data!(k, |k| {
let k: Box<dyn Events> = Box::new(k);
info!("-------------------------\ngot len {}", k.len());
trace!("got len {}", k.len());
let k = tr.0.transform(k);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k)))
})
@@ -89,7 +89,7 @@ fn timebinned_to_collectable(
let stream = stream.map(|k| {
on_sitemty_data!(k, |k| {
let k: Box<dyn Collectable> = Box::new(k);
info!("-------------------------\ngot len {}", k.len());
trace!("got len {}", k.len());
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k)))
})
});
@@ -98,7 +98,6 @@ fn timebinned_to_collectable(
}
pub async fn timebinned_json(query: BinnedQuery, _chconf: ChConf, cluster: Cluster) -> Result<JsonValue, Error> {
info!("~~~~~~~~~~~ timebinned_json");
let deadline = Instant::now().checked_add(query.timeout_value()).unwrap();
let binned_range = BinnedRangeEnum::covering_range(query.range().clone(), query.bin_count())?;
let collect_max = 10000;