daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adapt tests for moved test data

Browse Source
This commit is contained in:
Dominik Werder
2022-02-17 12:55:34 +01:00
parent 264c4307b9
commit 47acaa68a0
14 changed files with 492 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
disk/src/aggtest.rs
View File

@@ -1,6 +1,7 @@
use crate::eventblobs::EventChunkerMultifile;
use crate::eventchunker::EventChunkerConf;
use netpod::{timeunits::*, FileIoBufferSize};
use netpod::timeunits::*;
use netpod::{test_data_base_path_databuffer, FileIoBufferSize};
use netpod::{ByteOrder, ByteSize, Channel, ChannelConfig, NanoRange, Nanos, Node, ScalarType, Shape};
#[allow(unused_imports)]
use tracing::{debug, error, info, trace, warn};
@@ -12,8 +13,8 @@ pub fn make_test_node(id: u32) -> Node {
port: 8800 + id as u16,
port_raw: 8800 + id as u16 + 100,
// TODO use a common function to supply the tmp path.
data_base_path: format!("../tmpdata/node{:02}", id).into(),
cache_base_path: format!("../tmpdata/node{:02}", id).into(),
data_base_path: test_data_base_path_databuffer().join(format!("node{:02}", id)),
cache_base_path: test_data_base_path_databuffer().join(format!("node{:02}", id)),
ksprefix: "ks".into(),
backend: "testbackend".into(),
splits: None,

13
disk/src/binned.rs
View File

@@ -14,17 +14,17 @@ use bytes::Bytes;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::frame::MakeBytesFrame;
use items::numops::NumOps;
use items::streams::{Collectable, Collector};
use items::{
Clearable, EventsNodeProcessor, FilterFittingInside, Framable, FrameType, PushableIndex, RangeCompletableItem,
Sitemty, StreamItem, TimeBinnableType, WithLen,
};
use netpod::log::*;
use netpod::query::{BinnedQuery, RawEventsQuery};
use netpod::{log::*, ScalarType};
use netpod::{
x_bin_count, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, Shape,
x_bin_count, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, ScalarType,
Shape,
};
use serde::de::DeserializeOwned;
use std::fmt::Debug;
@@ -173,6 +173,13 @@ impl<S> BinnedBytesForHttpStream<S> {
}
}
pub trait MakeBytesFrame {
fn make_bytes_frame(&self) -> Result<Bytes, Error> {
// TODO only implemented for one type, remove
err::todoval()
}
}
impl<S, I> Stream for BinnedBytesForHttpStream<S>
where
S: Stream<Item = I> + Unpin,

156
disk/src/dataopen.rs
View File

@@ -401,69 +401,31 @@ async fn open_expanded_files_inner(
Ok(())
}
#[test]
fn expanded_file_list() {
use netpod::timeunits::*;
let range = netpod::NanoRange {
beg: DAY + HOUR * 5,
end: DAY + HOUR * 8,
};
let chn = netpod::Channel {
backend: "testbackend".into(),
name: "scalar-i32-be".into(),
};
// TODO read config from disk.
let channel_config = ChannelConfig {
channel: chn,
keyspace: 2,
time_bin_size: Nanos { ns: DAY },
scalar_type: netpod::ScalarType::I32,
byte_order: netpod::ByteOrder::big_endian(),
shape: netpod::Shape::Scalar,
array: false,
compression: false,
};
let cluster = netpod::test_cluster();
let task = async move {
let mut paths = vec![];
let mut files = open_expanded_files(&range, &channel_config, cluster.nodes[0].clone());
while let Some(file) = files.next().await {
match file {
Ok(k) => {
debug!("opened file: {:?}", k);
paths.push(k.files);
}
Err(e) => {
error!("error while trying to open {:?}", e);
break;
}
}
}
if paths.len() != 2 {
return Err(Error::with_msg_no_trace("expected 2 files"));
}
Ok(())
};
taskrun::run(task).unwrap();
}
#[cfg(test)]
mod test {
use crate::dataopen::position_file;
use super::*;
use err::Error;
use netpod::timeunits::{DAY, HOUR, MS};
use netpod::NanoRange;
use netpod::timeunits::*;
use netpod::{test_data_base_path_databuffer, ChannelConfig, NanoRange, Nanos};
use std::path::PathBuf;
use tokio::fs::OpenOptions;
const WAVE_FILE: &str =
"../tmpdata/node00/ks_3/byTime/wave-f64-be-n21/0000000000000000001/0000000000/0000000000086400000_00000_Data";
const SCALAR_FILE: &str =
"../tmpdata/node00/ks_2/byTime/scalar-i32-be/0000000000000000001/0000000000/0000000000086400000_00000_Data";
fn scalar_file_path() -> PathBuf {
test_data_base_path_databuffer()
.join("node00/ks_2/byTime/scalar-i32-be")
.join("0000000000000000001/0000000000/0000000000086400000_00000_Data")
}
fn wave_file_path() -> PathBuf {
test_data_base_path_databuffer()
.join("node00/ks_3/byTime/wave-f64-be-n21")
.join("0000000000000000001/0000000000/0000000000086400000_00000_Data")
}
#[test]
fn position_basic_file_at_begin() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY,
end: DAY + MS * 20000,
@@ -482,7 +444,7 @@ mod test {
#[test]
fn position_basic_file_for_empty_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 80000,
end: DAY + MS * 80000,
@@ -500,7 +462,7 @@ mod test {
#[test]
fn position_basic_file_at_begin_for_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY,
end: DAY + MS * 300000,
@@ -519,7 +481,7 @@ mod test {
#[test]
fn position_basic_file_at_inner() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 4000,
end: DAY + MS * 7000,
@@ -539,7 +501,7 @@ mod test {
#[test]
fn position_basic_file_at_inner_for_too_small_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 1501,
end: DAY + MS * 1502,
@@ -558,7 +520,7 @@ mod test {
#[test]
fn position_basic_file_starts_after_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: HOUR * 22,
end: HOUR * 23,
@@ -576,7 +538,7 @@ mod test {
#[test]
fn position_basic_file_ends_before_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY * 2,
end: DAY * 2 + HOUR,
@@ -594,7 +556,7 @@ mod test {
#[test]
fn position_basic_index() -> Result<(), Error> {
let fut = async {
let path = WAVE_FILE.into();
let path = wave_file_path();
let range = NanoRange {
beg: DAY + MS * 4000,
end: DAY + MS * 90000,
@@ -613,7 +575,7 @@ mod test {
#[test]
fn position_basic_index_too_small_range() -> Result<(), Error> {
let fut = async {
let path = WAVE_FILE.into();
let path = wave_file_path();
let range = NanoRange {
beg: DAY + MS * 3100,
end: DAY + MS * 3200,
@@ -631,7 +593,7 @@ mod test {
#[test]
fn position_basic_index_starts_after_range() -> Result<(), Error> {
let fut = async {
let path = WAVE_FILE.into();
let path = wave_file_path();
let range = NanoRange {
beg: HOUR * 10,
end: HOUR * 12,
@@ -649,7 +611,7 @@ mod test {
#[test]
fn position_basic_index_ends_before_range() -> Result<(), Error> {
let fut = async {
let path = WAVE_FILE.into();
let path = wave_file_path();
let range = NanoRange {
beg: DAY * 2,
end: DAY * 2 + MS * 40000,
@@ -672,7 +634,7 @@ mod test {
#[test]
fn position_expand_file_at_begin_no_fallback() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE;
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 3000,
end: DAY + MS * 40000,
@@ -692,7 +654,7 @@ mod test {
#[test]
fn position_expand_left_file_at_evts_file_begin() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY,
end: DAY + MS * 40000,
@@ -710,7 +672,7 @@ mod test {
#[test]
fn position_expand_right_file_at_evts_file_begin() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY,
end: DAY + MS * 40000,
@@ -729,7 +691,7 @@ mod test {
#[test]
fn position_expand_left_file_at_evts_file_within() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 3000,
end: DAY + MS * 40000,
@@ -749,7 +711,7 @@ mod test {
#[test]
fn position_expand_left_file_ends_before_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY * 2,
end: DAY * 2 + MS * 40000,
@@ -769,7 +731,7 @@ mod test {
#[test]
fn position_expand_left_file_begins_exactly_after_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: HOUR * 23,
end: DAY,
@@ -788,7 +750,7 @@ mod test {
#[test]
fn position_expand_right_file_begins_exactly_after_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: HOUR * 23,
end: DAY,
@@ -808,7 +770,7 @@ mod test {
#[test]
fn position_expand_left_basic_file_at_inner_for_too_small_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 1501,
end: DAY + MS * 1502,
@@ -828,7 +790,7 @@ mod test {
#[test]
fn position_expand_right_basic_file_at_inner_for_too_small_range() -> Result<(), Error> {
let fut = async {
let path = SCALAR_FILE.into();
let path = scalar_file_path();
let range = NanoRange {
beg: DAY + MS * 1501,
end: DAY + MS * 1502,
@@ -843,4 +805,52 @@ mod test {
taskrun::run(fut)?;
Ok(())
}
#[test]
fn expanded_file_list() {
let range = netpod::NanoRange {
beg: DAY + HOUR * 5,
end: DAY + HOUR * 8,
};
let chn = netpod::Channel {
backend: "testbackend".into(),
name: "scalar-i32-be".into(),
};
// TODO read config from disk? Or expose the config from data generator?
let channel_config = ChannelConfig {
channel: chn,
keyspace: 2,
time_bin_size: Nanos { ns: DAY },
scalar_type: netpod::ScalarType::I32,
byte_order: netpod::ByteOrder::big_endian(),
shape: netpod::Shape::Scalar,
array: false,
compression: false,
};
let cluster = netpod::test_cluster();
let task = async move {
let mut paths = vec![];
let mut files = open_expanded_files(&range, &channel_config, cluster.nodes[0].clone());
while let Some(file) = files.next().await {
match file {
Ok(k) => {
debug!("opened file: {:?}", k);
paths.push(k.files);
}
Err(e) => {
error!("error while trying to open {:?}", e);
break;
}
}
}
if paths.len() != 2 {
return Err(Error::with_msg_no_trace(format!(
"expected 2 files got {n}",
n = paths.len()
)));
}
Ok(())
};
taskrun::run(task).unwrap();
}
}

20
disk/src/merge.rs
View File

@@ -296,17 +296,25 @@ mod test {
use err::Error;
use futures_util::StreamExt;
use items::{RangeCompletableItem, StreamItem};
use netpod::log::*;
use netpod::timeunits::{DAY, MS};
use netpod::{log::*, test_data_base_path_databuffer};
use netpod::{ByteOrder, ByteSize, Channel, ChannelConfig, FileIoBufferSize, NanoRange, Nanos, ScalarType, Shape};
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
const SCALAR_FILE: &str =
"../tmpdata/node00/ks_2/byTime/scalar-i32-be/0000000000000000001/0000000000/0000000000086400000_00000_Data";
const _WAVE_FILE: &str =
"../tmpdata/node00/ks_3/byTime/wave-f64-be-n21/0000000000000000001/0000000000/0000000000086400000_00000_Data";
fn scalar_file_path() -> PathBuf {
test_data_base_path_databuffer()
.join("node00/ks_2/byTime/scalar-i32-be")
.join("0000000000000000001/0000000000/0000000000086400000_00000_Data")
}
#[allow(unused)]
fn wave_file_path() -> PathBuf {
test_data_base_path_databuffer()
.join("node00/ks_3/byTime/wave-f64-be-n21")
.join("0000000000000000001/0000000000/0000000000086400000_00000_Data")
}
#[derive(Debug)]
struct CollectedEvents {
@@ -399,7 +407,7 @@ mod test {
beg: DAY + MS * 1501,
end: DAY + MS * 4000,
};
let path = PathBuf::from(SCALAR_FILE);
let path = scalar_file_path();
collect_merged_events(vec![path], range).await?;
// TODO