daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2023-03-16 10:10:16 +01:00
parent b0f71b2143
commit 6a86ac1063
46 changed files with 549 additions and 1871 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
daqbufp2/src/client.rs
View File

@@ -11,7 +11,6 @@ use items_0::streamitem::StreamItem;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::query::CacheUsage;
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::ByteSize;
use netpod::Channel;
@@ -67,9 +66,9 @@ pub async fn get_binned(
name: channel_name.into(),
series: None,
};
let agg_kind = AggKind::DimXBins1;
let range = NanoRange::from_date_time(beg_date, end_date);
let mut query = BinnedQuery::new(channel, range, bin_count);
let range = NanoRange::from_date_time(beg_date, end_date).into();
// TODO this was before fixed using AggKind::DimXBins1
let mut query = BinnedQuery::new(channel, range, bin_count).for_time_weighted_scalar();
query.set_cache_usage(cache_usage);
query.set_disk_stats_every(ByteSize(1024 * disk_stats_every_kb));
let hp = HostPort { host: host, port: port };

18
daqbufp2/src/test/api4/binnedjson.rs
View File

@@ -1,14 +1,18 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use crate::test::f32_cmp_near;
use chrono::{DateTime, Utc};
use chrono::Utc;
use err::Error;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
use netpod::HostPort;
use netpod::NanoRange;
use netpod::APP_JSON;
use netpod::{log::*, AggKind};
use netpod::{AppendToUrl, Channel, Cluster, HostPort, NanoRange};
use serde_json::Value as JsonValue;
use url::Url;
@@ -267,10 +271,10 @@ async fn get_binned_json(
) -> Result<JsonValue, Error> {
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date);
let query = BinnedQuery::new(channel, range, bin_count, Some(AggKind::TimeWeightedScalar));
let beg_date = beg_date.parse()?;
let end_date = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date).into();
let query = BinnedQuery::new(channel, range, bin_count).for_time_weighted_scalar();
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", hp.host, hp.port))?;
query.append_to_url(&mut url);

16
daqbufp2/src/test/api4/eventsjson.rs
View File

@@ -1,14 +1,18 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use crate::test::f32_iter_cmp_near;
use chrono::{DateTime, Utc};
use chrono::Utc;
use err::Error;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::query::PlainEventsQuery;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
use netpod::HostPort;
use netpod::NanoRange;
use netpod::APP_JSON;
use netpod::{log::*, AggKind};
use netpod::{AppendToUrl, Channel, Cluster, HostPort, NanoRange};
use serde_json::Value as JsonValue;
use url::Url;
@@ -100,10 +104,10 @@ async fn events_plain_json(
) -> Result<JsonValue, Error> {
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let beg_date = beg_date.parse()?;
let end_date = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range, Some(AggKind::TimeWeightedScalar), None, None);
let query = PlainEventsQuery::new(channel, range).for_time_weighted_scalar();
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);

12
daqbufp2/src/test/binnedbinary.rs
View File

@@ -1,6 +1,5 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use chrono::DateTime;
use chrono::Utc;
use disk::streamlog::Streamlog;
use err::Error;
@@ -14,7 +13,6 @@ use items_0::subfr::SubFrId;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::query::CacheUsage;
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
@@ -104,10 +102,9 @@ where
NTY: fmt::Debug + SubFrId + DeserializeOwned,
{
let t1 = Utc::now();
let agg_kind = AggKind::DimXBins1;
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let beg_date = beg_date.parse()?;
let end_date = end_date.parse()?;
let channel_backend = "testbackend";
let perf_opts = PerfOpts::default();
let channel = Channel {
@@ -115,8 +112,9 @@ where
name: channel_name.into(),
series: None,
};
let range = NanoRange::from_date_time(beg_date, end_date);
let mut query = BinnedQuery::new(channel, range, bin_count, Some(agg_kind));
let range = NanoRange::from_date_time(beg_date, end_date).into();
// TODO before, these tests were all fixed using AggKind::DimXBins1
let mut query = BinnedQuery::new(channel, range, bin_count).for_time_weighted_scalar();
query.set_cache_usage(CacheUsage::Ignore);
query.set_buf_len_disk_io(1024 * 16);
let hp = HostPort::from_node(node0);

5
daqbufp2/src/test/events.rs
View File

@@ -12,7 +12,6 @@ use hyper::Body;
use items_0::streamitem::StreamItem;
use netpod::log::*;
use netpod::query::PlainEventsQuery;
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
@@ -87,7 +86,7 @@ async fn get_plain_events_binary(
series: None,
};
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range, Some(AggKind::TimeWeightedScalar), None, None);
let query = PlainEventsQuery::new(channel, range).for_time_weighted_scalar();
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);
@@ -260,7 +259,7 @@ pub async fn get_plain_events_json(
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range, Some(AggKind::TimeWeightedScalar), None, None);
let query = PlainEventsQuery::new(channel, range);
let hp = HostPort::from_node(node0);
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
query.append_to_url(&mut url);

28
daqbufp2/src/test/timeweightedjson.rs
View File

@@ -1,12 +1,18 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use chrono::{DateTime, Utc};
use chrono::DateTime;
use chrono::Utc;
use err::Error;
use http::StatusCode;
use hyper::Body;
use netpod::query::{BinnedQuery, CacheUsage};
use netpod::{log::*, AppendToUrl};
use netpod::{AggKind, Channel, Cluster, NanoRange, APP_JSON};
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::query::CacheUsage;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
use netpod::NanoRange;
use netpod::APP_JSON;
use std::time::Duration;
use url::Url;
@@ -20,7 +26,7 @@ fn time_weighted_json_03() -> Result<(), Error> {
"1970-01-01T00:20:11.000Z",
"1970-01-01T00:30:20.000Z",
10,
AggKind::TimeWeightedScalar,
//AggKind::TimeWeightedScalar,
cluster,
11,
true,
@@ -36,6 +42,7 @@ fn time_weighted_json_03() -> Result<(), Error> {
#[test]
fn time_weighted_json_10() -> Result<(), Error> {
async fn inner() -> Result<(), Error> {
error!("TODO this test asked for DimXBins1");
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
get_json_common(
@@ -43,7 +50,7 @@ fn time_weighted_json_10() -> Result<(), Error> {
"1970-01-01T00:20:10.000Z",
"1970-01-01T01:20:30.000Z",
10,
AggKind::DimXBins1,
//AggKind::DimXBins1,
cluster,
13,
true,
@@ -64,7 +71,7 @@ fn time_weighted_json_20() -> Result<(), Error> {
"1970-01-01T00:20:10.000Z",
"1970-01-01T01:20:45.000Z",
10,
AggKind::TimeWeightedScalar,
//AggKind::TimeWeightedScalar,
cluster,
13,
true,
@@ -86,7 +93,8 @@ async fn get_json_common(
beg_date: &str,
end_date: &str,
bin_count: u32,
agg_kind: AggKind,
// TODO refactor for Transform
//agg_kind: AggKind,
cluster: &Cluster,
expect_bin_count: u32,
expect_finalised_range: bool,
@@ -101,8 +109,8 @@ async fn get_json_common(
name: channel_name.into(),
series: None,
};
let range = NanoRange::from_date_time(beg_date, end_date);
let mut query = BinnedQuery::new(channel, range, bin_count, Some(agg_kind));
let range = NanoRange::from_date_time(beg_date, end_date).into();
let mut query = BinnedQuery::new(channel, range, bin_count).for_time_weighted_scalar();
query.set_timeout(Duration::from_millis(40000));
query.set_cache_usage(CacheUsage::Ignore);
let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", node0.host, node0.port))?;