Basic time-weighted binning

daqbufp2/src/lib.rs

@@ -1,13 +1,12 @@
-use tokio::task::JoinHandle;
-
-use err::Error;
-use netpod::{Cluster, NodeConfig, NodeConfigCached, ProxyConfig};
-
 pub mod client;
 pub mod nodes;
 #[cfg(test)]
 pub mod test;
 
+use err::Error;
+use netpod::{Cluster, NodeConfig, NodeConfigCached, ProxyConfig};
+use tokio::task::JoinHandle;
+
 pub fn spawn_test_hosts(cluster: Cluster) -> Vec<JoinHandle<Result<(), Error>>> {
     let mut ret = vec![];
     for node in &cluster.nodes {

daqbufp2/src/nodes.rs

@@ -1,6 +1,6 @@
 use crate::spawn_test_hosts;
 use err::Error;
-use netpod::{Cluster, Database, Node};
+use netpod::Cluster;
 use std::sync::{Arc, Mutex};
 use tokio::task::JoinHandle;
 

daqbufp2/src/test.rs

@@ -1,8 +1,19 @@
-use bytes::BytesMut;
-
 pub mod binnedbinary;
 pub mod binnedjson;
 pub mod events;
+pub mod timeweightedjson;
+
+use bytes::BytesMut;
+use err::Error;
+use std::future::Future;
+
+fn run_test<F>(f: F)
+where
+    F: Future<Output = Result<(), Error>>,
+{
+    std::env::set_current_dir("..").unwrap();
+    taskrun::run(f).unwrap();
+}
 
 #[test]
 fn bufs() {

daqbufp2/src/test/timeweightedjson.rs

@@ -0,0 +1,132 @@
+use crate::nodes::require_test_hosts_running;
+use chrono::{DateTime, Utc};
+use disk::binned::query::{BinnedQuery, CacheUsage};
+use err::Error;
+use http::StatusCode;
+use hyper::Body;
+use netpod::log::*;
+use netpod::{AggKind, AppendToUrl, Channel, Cluster, NanoRange, APP_JSON};
+use std::time::Duration;
+use url::Url;
+
+#[test]
+fn time_weighted_json_0() {
+    super::run_test(time_weighted_json_0_inner());
+}
+
+async fn time_weighted_json_0_inner() -> Result<(), Error> {
+    let rh = require_test_hosts_running()?;
+    let cluster = &rh.cluster;
+    get_json_common(
+        "scalar-i32-be",
+        "1970-01-01T00:20:10.000Z",
+        "1970-01-01T01:20:30.000Z",
+        10,
+        AggKind::TimeWeightedScalar,
+        cluster,
+        13,
+        true,
+    )
+    .await
+}
+
+#[test]
+fn time_weighted_json_1() {
+    super::run_test(time_weighted_json_1_inner());
+}
+
+async fn time_weighted_json_1_inner() -> Result<(), Error> {
+    let rh = require_test_hosts_running()?;
+    let cluster = &rh.cluster;
+    get_json_common(
+        "wave-f64-be-n21",
+        "1970-01-01T00:20:10.000Z",
+        "1970-01-01T01:20:45.000Z",
+        10,
+        AggKind::TimeWeightedScalar,
+        cluster,
+        13,
+        true,
+    )
+    .await
+}
+
+// For waveform with N x-bins, see test::binnedjson
+
+async fn get_json_common(
+    channel_name: &str,
+    beg_date: &str,
+    end_date: &str,
+    bin_count: u32,
+    agg_kind: AggKind,
+    cluster: &Cluster,
+    expect_bin_count: u32,
+    expect_finalised_range: bool,
+) -> Result<(), Error> {
+    let t1 = Utc::now();
+    let node0 = &cluster.nodes[0];
+    let beg_date: DateTime<Utc> = beg_date.parse()?;
+    let end_date: DateTime<Utc> = end_date.parse()?;
+    let channel_backend = "testbackend";
+    let channel = Channel {
+        backend: channel_backend.into(),
+        name: channel_name.into(),
+    };
+    let range = NanoRange::from_date_time(beg_date, end_date);
+    let mut query = BinnedQuery::new(channel, range, bin_count, agg_kind);
+    query.set_timeout(Duration::from_millis(10000));
+    query.set_cache_usage(CacheUsage::Ignore);
+    let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", node0.host, node0.port))?;
+    query.append_to_url(&mut url);
+    let url = url;
+    info!("get_json_common  get {}", url);
+    let req = hyper::Request::builder()
+        .method(http::Method::GET)
+        .uri(url.to_string())
+        .header(http::header::ACCEPT, APP_JSON)
+        .body(Body::empty())?;
+    let client = hyper::Client::new();
+    let res = client.request(req).await?;
+    if res.status() != StatusCode::OK {
+        error!("get_json_common client response {:?}", res);
+    }
+    let res = hyper::body::to_bytes(res.into_body()).await?;
+    let t2 = chrono::Utc::now();
+    let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
+    info!("get_json_common  DONE  time {} ms", ms);
+    let res = String::from_utf8_lossy(&res).to_string();
+    //info!("get_json_common res: {}", res);
+    let res: serde_json::Value = serde_json::from_str(res.as_str())?;
+    info!(
+        "result from endpoint: --------------\n{}\n--------------",
+        serde_json::to_string_pretty(&res)?
+    );
+    // TODO enable in future:
+    if false {
+        if expect_finalised_range {
+            if !res
+                .get("finalisedRange")
+                .ok_or(Error::with_msg("missing finalisedRange"))?
+                .as_bool()
+                .ok_or(Error::with_msg("key finalisedRange not bool"))?
+            {
+                return Err(Error::with_msg("expected finalisedRange"));
+            }
+        } else if res.get("finalisedRange").is_some() {
+            return Err(Error::with_msg("expect absent finalisedRange"));
+        }
+    }
+    if res.get("counts").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
+        return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
+    }
+    if res.get("mins").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
+        return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
+    }
+    if res.get("maxs").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
+        return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
+    }
+    if res.get("avgs").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
+        return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
+    }
+    Ok(())
+}