daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix tests

Browse Source
This commit is contained in:
Dominik Werder
2023-05-04 12:00:02 +02:00
parent 03854395ff
commit 8fc73fd6ed
22 changed files with 66 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
daqbufp2/src/test.rs
View File

@@ -5,8 +5,6 @@ mod api4;
pub mod archapp;
pub mod binnedjson;
#[cfg(test)]
mod events;
#[cfg(test)]
mod timeweightedjson;
use bytes::BytesMut;

2
daqbufp2/src/test/api1.rs
View File

@@ -60,7 +60,7 @@ fn events_f64_plain() -> Result<(), Error> {
let accept = "application/octet-stream";
let range = Api1Range::new("1970-01-01T00:00:00Z".try_into()?, "1970-01-01T00:01:00Z".try_into()?)?;
// TODO the channel list needs to get pre-processed to check for backend prefix!
let ch = ChannelTuple::new(TEST_BACKEND.into(), "scalar-i32-be".into());
let ch = ChannelTuple::new(TEST_BACKEND.into(), "test-gen-i32-dim0-v01".into());
let qu = Api1Query::new(range, vec![ch]);
let body = serde_json::to_string(&qu)?;
let buf = http_post(url, accept, body.into()).await?;

53
daqbufp2/src/test/api4/binnedjson.rs
View File

@@ -364,57 +364,6 @@ fn binned_d0_json_04() -> Result<(), Error> {
taskrun::run(fut)
}
#[test]
fn binned_inmem_d0_json_00() -> Result<(), Error> {
let fut = async {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
let query = make_query(
"inmem-d0-i32",
"1970-01-01T00:20:04.000Z",
"1970-01-01T00:21:10.000Z",
10,
)?;
let jsv = fetch_binned_json(query, cluster).await?;
let res: BinsDim0CollectedResult<i32> = serde_json::from_value(jsv)?;
assert_eq!(res.range_final(), true);
assert_eq!(res.timed_out(), false);
assert_eq!(res.len(), 14);
assert_eq!(res.ts_anchor_sec(), 1200);
{
let v1: Vec<_> = res.counts().iter().map(|x| *x).collect();
assert_eq!(&v1, &[5; 14]);
}
{
let v1: Vec<_> = res.ts1_off_ms().iter().map(|x| *x).collect();
let v2: Vec<_> = (0..14).into_iter().map(|x| 5000 * x).collect();
assert_eq!(&v1, &v2);
}
{
let v1: Vec<_> = res.ts2_off_ms().iter().map(|x| *x).collect();
let v2: Vec<_> = (1..15).into_iter().map(|x| 5000 * x).collect();
assert_eq!(&v1, &v2);
}
{
let v1: Vec<_> = res.mins().iter().map(|x| *x).collect();
let v2: Vec<_> = (0..14).into_iter().map(|x| 1200 + 5 * x).collect();
assert_eq!(&v1, &v2);
}
{
let v1: Vec<_> = res.maxs().iter().map(|x| *x).collect();
let v2: Vec<_> = (0..14).into_iter().map(|x| 1204 + 5 * x).collect();
assert_eq!(&v1, &v2);
}
{
let v1: Vec<_> = res.avgs().iter().map(|x| *x).collect();
let v2: Vec<_> = (0..14).into_iter().map(|x| 1202. + 5. * x as f32).collect();
assert_eq!(f32_iter_cmp_near(v1, v2, 0.05, 0.05), true);
}
Ok(())
};
taskrun::run(fut)
}
async fn get_binned_json(
channel: Channel,
beg_date: &str,
@@ -453,7 +402,7 @@ async fn get_binned_json(
let s = String::from_utf8_lossy(&buf);
let res: JsonValue = serde_json::from_str(&s)?;
let pretty = serde_json::to_string_pretty(&res)?;
info!("Received from remote:\n{pretty}");
debug!("get_binned_json pretty {pretty}");
let t2 = chrono::Utc::now();
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
// TODO add timeout

4
daqbufp2/src/test/api4/common.rs
View File

@@ -21,7 +21,7 @@ pub async fn fetch_events_json(query: PlainEventsQuery, cluster: &Cluster) -> Re
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);
let url = url;
info!("http get {}", url);
debug!("fetch_events_json url {}", url);
let req = hyper::Request::builder()
.method(http::Method::GET)
.uri(url.to_string())
@@ -54,7 +54,7 @@ pub async fn fetch_binned_json(query: BinnedQuery, cluster: &Cluster) -> Result<
let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", hp.host, hp.port))?;
query.append_to_url(&mut url);
let url = url;
info!("http get {}", url);
debug!("fetch_binned_json url {}", url);
let req = hyper::Request::builder()
.method(http::Method::GET)
.uri(url.to_string())

10
daqbufp2/src/test/api4/eventsjson.rs
View File

@@ -20,12 +20,7 @@ use url::Url;
const TEST_BACKEND: &str = "testbackend-00";
fn make_query<S: Into<String>>(
name: S,
beg_date: &str,
end_date: &str,
//bin_count_min: u32,
) -> Result<PlainEventsQuery, Error> {
fn make_query<S: Into<String>>(name: S, beg_date: &str, end_date: &str) -> Result<PlainEventsQuery, Error> {
let channel = Channel {
backend: TEST_BACKEND.into(),
name: name.into(),
@@ -50,7 +45,7 @@ fn events_plain_json_00() -> Result<(), Error> {
)?;
let jsv = fetch_events_json(query, cluster).await?;
let res: EventsDim0CollectorOutput<i32> = serde_json::from_value(jsv)?;
// Tim-weighted will use one event before:
// Tim-weighted uses one event before requested range:
assert_eq!(res.len(), 133);
assert_eq!(res.ts_anchor_sec(), 1203);
Ok(())
@@ -114,7 +109,6 @@ async fn events_plain_json(
}
let buf = hyper::body::to_bytes(res.into_body()).await.ec()?;
let s = String::from_utf8_lossy(&buf);
//info!("received from server: {s}");
let res: JsonValue = serde_json::from_str(&s)?;
let pretty = serde_json::to_string_pretty(&res)?;
info!("{pretty}");

3
daqbufp2/src/test/archapp.rs
View File

@@ -1,7 +1,5 @@
#![allow(unused)]
use super::events::get_plain_events_json;
use crate::nodes::require_archapp_test_host_running;
use crate::test::events::ch_gen;
use err::Error;
use netpod::f64_close;
use netpod::log::*;
@@ -18,6 +16,7 @@ fn get_events_1() -> Result<(), Error> {
let rh = require_archapp_test_host_running()?;
let cluster = &rh.cluster;
let res = get_plain_events_json(
// TODO this just added test backend name, no series id.
ch_gen("SARUN16-MQUA080:X"),
"2021-01-04T00:00:00Z",
"2021-01-30T00:00:00Z",

296
daqbufp2/src/test/events.rs
View File

@@ -1,296 +0,0 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use chrono::DateTime;
use chrono::Utc;
use disk::streamlog::Streamlog;
use err::Error;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use http::StatusCode;
use httpclient::HttpBodyAsAsyncRead;
use hyper::Body;
use items_0::streamitem::StreamItem;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
use netpod::HostPort;
use netpod::PerfOpts;
use netpod::APP_JSON;
use netpod::APP_OCTET;
use query::api4::events::PlainEventsQuery;
use serde_json::Value as JsonValue;
use std::fmt::Debug;
use std::future::ready;
use streams::frames::inmem::InMemoryFrameAsyncReadStream;
use tokio::io::AsyncRead;
use url::Url;
const TEST_BACKEND: &str = "testbackend-00";
fn ch_adhoc(name: &str) -> Channel {
Channel {
series: None,
backend: TEST_BACKEND.into(),
name: name.into(),
}
}
pub fn ch_gen(name: &str) -> Channel {
Channel {
series: None,
backend: TEST_BACKEND.into(),
name: name.into(),
}
}
// TODO OFFENDING TEST add actual checks on result
async fn get_plain_events_binary_0_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
if true {
get_plain_events_binary(
"scalar-i32-be",
"1970-01-01T00:20:10.000Z",
"1970-01-01T00:20:50.000Z",
cluster,
true,
4,
)
.await?;
}
Ok(())
}
#[test]
fn get_plain_events_binary_0() {
taskrun::run(get_plain_events_binary_0_inner()).unwrap();
}
async fn get_plain_events_binary(
channel_name: &str,
beg_date: &str,
end_date: &str,
cluster: &Cluster,
_expect_range_complete: bool,
_expect_event_count: u64,
) -> Result<EventsResponse, Error> {
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let channel_backend = TEST_BACKEND;
let perf_opts = PerfOpts::default();
let channel = Channel {
backend: channel_backend.into(),
name: channel_name.into(),
series: None,
};
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range).for_time_weighted_scalar();
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);
let url = url;
debug!("get_plain_events_binary get {}", url);
let req = hyper::Request::builder()
.method(http::Method::GET)
.uri(url.to_string())
.header(http::header::ACCEPT, APP_OCTET)
.body(Body::empty())
.ec()?;
let client = hyper::Client::new();
let res = client.request(req).await.ec()?;
if res.status() != StatusCode::OK {
error!("client response {res:?}");
return Err(format!("get_plain_events_binary client response {res:?}").into());
}
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
let res = consume_plain_events_binary(s2).await?;
let t2 = chrono::Utc::now();
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
// TODO add timeout
debug!("get_plain_events_binary time {} ms", ms);
if !res.is_valid() {
Ok(res)
} else {
Ok(res)
}
}
#[allow(unused)]
#[derive(Debug)]
pub struct EventsResponse {
event_count: u64,
err_item_count: u64,
data_item_count: u64,
bytes_read: u64,
range_complete_count: u64,
log_item_count: u64,
#[allow(unused)]
stats_item_count: u64,
}
impl EventsResponse {
pub fn new() -> Self {
Self {
event_count: 0,
err_item_count: 0,
data_item_count: 0,
bytes_read: 0,
range_complete_count: 0,
log_item_count: 0,
stats_item_count: 0,
}
}
pub fn is_valid(&self) -> bool {
if self.range_complete_count > 1 {
false
} else {
true
}
}
}
async fn consume_plain_events_binary<T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<EventsResponse, Error>
where
T: AsyncRead + Unpin,
{
let s1 = inp
.map_err(|e| error!("TEST GOT ERROR {:?}", e))
.filter_map(|item| {
let g = match item {
Ok(item) => match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
None
}
StreamItem::Stats(item) => {
debug!("Stats: {:?}", item);
None
}
StreamItem::DataItem(_frame) => {
err::todo();
Some(Ok(()))
}
},
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
};
ready(g)
})
.fold(EventsResponse::new(), |a, _x| ready(a));
let ret = s1.await;
debug!("result: {:?}", ret);
Ok(ret)
}
async fn get_plain_events_json_0_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
get_plain_events_json(
ch_gen("scalar-i32-be"),
"1970-01-01T00:20:10.000Z",
"1970-01-01T00:20:12.000Z",
cluster,
true,
4,
)
.await?;
Ok(())
}
#[test]
fn get_plain_events_json_0() {
taskrun::run(get_plain_events_json_0_inner()).unwrap();
}
async fn get_plain_events_json_1_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
get_plain_events_json(
ch_gen("wave-f64-be-n21"),
"1970-01-01T00:20:10.000Z",
"1970-01-01T00:20:12.000Z",
cluster,
true,
4,
)
.await?;
Ok(())
}
#[test]
fn get_plain_events_json_1() {
taskrun::run(get_plain_events_json_1_inner()).unwrap();
}
async fn get_plain_events_json_2_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
get_plain_events_json(
ch_adhoc("inmem-d0-i32"),
"1970-01-01T00:20:04.000Z",
"1970-01-01T00:20:10.000Z",
cluster,
true,
4,
)
.await?;
Ok(())
}
#[test]
fn get_plain_events_json_2() {
taskrun::run(get_plain_events_json_2_inner()).unwrap();
}
// TODO improve by a more information-rich return type.
pub async fn get_plain_events_json(
channel: Channel,
beg_date: &str,
end_date: &str,
cluster: &Cluster,
_expect_range_complete: bool,
_expect_event_count: u64,
) -> Result<JsonValue, Error> {
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range);
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);
let url = url;
info!("get_plain_events get {}", url);
let req = hyper::Request::builder()
.method(http::Method::GET)
.uri(url.to_string())
.header(http::header::ACCEPT, APP_JSON)
.body(Body::empty())
.ec()?;
let client = hyper::Client::new();
let res = client.request(req).await.ec()?;
trace!("Response {res:?}");
if res.status() != StatusCode::OK {
error!("client response {:?}", res);
}
let buf = hyper::body::to_bytes(res.into_body()).await.ec()?;
let s = String::from_utf8_lossy(&buf);
let res: JsonValue = serde_json::from_str(&s)?;
eprintln!("res {res:?}");
// TODO assert more
let t2 = chrono::Utc::now();
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
// TODO add timeout
debug!("time {} ms", ms);
Ok(res)
}