Factor out query type

query/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "query"
+version = "0.0.1"
+authors = ["Dominik Werder <dominik.werder@gmail.com>"]
+edition = "2021"
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tracing = "0.1"
+chrono = { version = "0.4.19", features = ["serde"] }
+url = "2.2"
+humantime-serde = "1.1.1"
+err = { path = "../err" }
+netpod = { path = "../netpod" }
+items_0 = { path = "../items_0" }

query/src/api4.rs

@@ -0,0 +1,2 @@
+pub mod binned;
+pub mod events;

query/src/api4/binned.rs

@@ -0,0 +1,226 @@
+use crate::transform::TransformQuery;
+use err::Error;
+use netpod::get_url_query_pairs;
+use netpod::log::*;
+use netpod::query::CacheUsage;
+use netpod::query::PulseRangeQuery;
+use netpod::query::TimeRangeQuery;
+use netpod::range::evrange::SeriesRange;
+use netpod::AppendToUrl;
+use netpod::ByteSize;
+use netpod::Channel;
+use netpod::FromUrl;
+use netpod::HasBackend;
+use netpod::HasTimeout;
+use serde::Deserialize;
+use serde::Serialize;
+use std::collections::BTreeMap;
+use std::time::Duration;
+use url::Url;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct BinnedQuery {
+    channel: Channel,
+    range: SeriesRange,
+    bin_count: u32,
+    #[serde(
+        default = "TransformQuery::default_time_binned",
+        skip_serializing_if = "TransformQuery::is_default_time_binned"
+    )]
+    transform: TransformQuery,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    cache_usage: Option<CacheUsage>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    bins_max: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    timeout: Option<Duration>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    buf_len_disk_io: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    disk_stats_every: Option<ByteSize>,
+}
+
+impl BinnedQuery {
+    pub fn new(channel: Channel, range: SeriesRange, bin_count: u32) -> Self {
+        Self {
+            channel,
+            range,
+            bin_count,
+            transform: TransformQuery::default_time_binned(),
+            cache_usage: None,
+            bins_max: None,
+            buf_len_disk_io: None,
+            disk_stats_every: None,
+            timeout: None,
+        }
+    }
+
+    pub fn range(&self) -> &SeriesRange {
+        &self.range
+    }
+
+    pub fn channel(&self) -> &Channel {
+        &self.channel
+    }
+
+    pub fn bin_count(&self) -> u32 {
+        self.bin_count
+    }
+
+    pub fn transform(&self) -> &TransformQuery {
+        &self.transform
+    }
+
+    pub fn cache_usage(&self) -> CacheUsage {
+        self.cache_usage.as_ref().map_or(CacheUsage::Use, |x| x.clone())
+    }
+
+    pub fn disk_stats_every(&self) -> ByteSize {
+        match &self.disk_stats_every {
+            Some(x) => x.clone(),
+            None => ByteSize(1024 * 1024 * 4),
+        }
+    }
+
+    pub fn buf_len_disk_io(&self) -> usize {
+        match self.buf_len_disk_io {
+            Some(x) => x,
+            None => 1024 * 16,
+        }
+    }
+
+    pub fn timeout(&self) -> Option<Duration> {
+        self.timeout.clone()
+    }
+
+    pub fn timeout_value(&self) -> Duration {
+        match &self.timeout {
+            Some(x) => x.clone(),
+            None => Duration::from_millis(10000),
+        }
+    }
+
+    pub fn bins_max(&self) -> u32 {
+        self.bins_max.unwrap_or(1024)
+    }
+
+    pub fn set_series_id(&mut self, series: u64) {
+        self.channel.series = Some(series);
+    }
+
+    pub fn channel_mut(&mut self) -> &mut Channel {
+        &mut self.channel
+    }
+
+    pub fn set_cache_usage(&mut self, k: CacheUsage) {
+        self.cache_usage = Some(k);
+    }
+
+    pub fn set_disk_stats_every(&mut self, k: ByteSize) {
+        self.disk_stats_every = Some(k);
+    }
+
+    pub fn set_timeout(&mut self, k: Duration) {
+        self.timeout = Some(k);
+    }
+
+    pub fn set_buf_len_disk_io(&mut self, k: usize) {
+        self.buf_len_disk_io = Some(k);
+    }
+
+    pub fn for_time_weighted_scalar(self) -> Self {
+        err::todo();
+        self
+    }
+}
+
+impl HasBackend for BinnedQuery {
+    fn backend(&self) -> &str {
+        &self.channel.backend
+    }
+}
+
+impl HasTimeout for BinnedQuery {
+    fn timeout(&self) -> Duration {
+        self.timeout_value()
+    }
+}
+
+impl FromUrl for BinnedQuery {
+    fn from_url(url: &Url) -> Result<Self, Error> {
+        let pairs = get_url_query_pairs(url);
+        Self::from_pairs(&pairs)
+    }
+
+    fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
+        let range = if let Ok(x) = TimeRangeQuery::from_pairs(pairs) {
+            SeriesRange::TimeRange(x.into())
+        } else if let Ok(x) = PulseRangeQuery::from_pairs(pairs) {
+            SeriesRange::PulseRange(x.into())
+        } else {
+            return Err(Error::with_msg_no_trace("no series range in url"));
+        };
+        let ret = Self {
+            channel: Channel::from_pairs(&pairs)?,
+            range,
+            bin_count: pairs
+                .get("binCount")
+                .ok_or(Error::with_msg("missing binCount"))?
+                .parse()
+                .map_err(|e| Error::with_msg(format!("can not parse binCount {:?}", e)))?,
+            transform: TransformQuery::from_pairs(pairs)?,
+            cache_usage: CacheUsage::from_pairs(&pairs)?,
+            buf_len_disk_io: pairs
+                .get("bufLenDiskIo")
+                .map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
+            disk_stats_every: pairs
+                .get("diskStatsEveryKb")
+                .map(|k| k.parse().ok())
+                .unwrap_or(None)
+                .map(ByteSize::kb),
+            /*report_error: pairs
+            .get("reportError")
+            .map_or("false", |k| k)
+            .parse()
+            .map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,*/
+            timeout: pairs
+                .get("timeout")
+                .map(|x| x.parse::<u64>().map(Duration::from_millis).ok())
+                .unwrap_or(None),
+            bins_max: pairs.get("binsMax").map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
+        };
+        debug!("BinnedQuery::from_url  {:?}", ret);
+        Ok(ret)
+    }
+}
+
+impl AppendToUrl for BinnedQuery {
+    fn append_to_url(&self, url: &mut Url) {
+        match &self.range {
+            SeriesRange::TimeRange(k) => TimeRangeQuery::from(k).append_to_url(url),
+            SeriesRange::PulseRange(k) => PulseRangeQuery::from(k).append_to_url(url),
+        }
+        self.channel.append_to_url(url);
+        {
+            let mut g = url.query_pairs_mut();
+            g.append_pair("binCount", &format!("{}", self.bin_count));
+        }
+        self.transform.append_to_url(url);
+        let mut g = url.query_pairs_mut();
+        if let Some(x) = &self.cache_usage {
+            g.append_pair("cacheUsage", &x.query_param_value());
+        }
+        if let Some(x) = &self.timeout {
+            g.append_pair("timeout", &format!("{}", x.as_millis()));
+        }
+        if let Some(x) = self.bins_max {
+            g.append_pair("binsMax", &format!("{}", x));
+        }
+        if let Some(x) = self.buf_len_disk_io {
+            g.append_pair("bufLenDiskIo", &format!("{}", x));
+        }
+        if let Some(x) = &self.disk_stats_every {
+            g.append_pair("diskStatsEveryKb", &format!("{}", x.bytes() / 1024));
+        }
+    }
+}

query/src/api4/events.rs

@@ -0,0 +1,235 @@
+use crate::transform::TransformQuery;
+use err::Error;
+use netpod::get_url_query_pairs;
+use netpod::is_false;
+use netpod::log::*;
+use netpod::query::CacheUsage;
+use netpod::query::PulseRangeQuery;
+use netpod::query::TimeRangeQuery;
+use netpod::range::evrange::SeriesRange;
+use netpod::AppendToUrl;
+use netpod::ByteSize;
+use netpod::Channel;
+use netpod::FromUrl;
+use netpod::HasBackend;
+use netpod::HasTimeout;
+use serde::Deserialize;
+use serde::Serialize;
+use std::collections::BTreeMap;
+use std::time::Duration;
+use url::Url;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct PlainEventsQuery {
+    channel: Channel,
+    range: SeriesRange,
+    #[serde(default, skip_serializing_if = "is_false", rename = "oneBeforeRange")]
+    one_before_range: bool,
+    #[serde(
+        default = "TransformQuery::default_events",
+        skip_serializing_if = "TransformQuery::is_default_events"
+    )]
+    transform: TransformQuery,
+    #[serde(default, skip_serializing_if = "Option::is_none", with = "humantime_serde")]
+    timeout: Option<Duration>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    events_max: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none", with = "humantime_serde")]
+    event_delay: Option<Duration>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    stream_batch_len: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    buf_len_disk_io: Option<usize>,
+    #[serde(default, skip_serializing_if = "is_false")]
+    do_test_main_error: bool,
+    #[serde(default, skip_serializing_if = "is_false")]
+    do_test_stream_error: bool,
+}
+
+impl PlainEventsQuery {
+    pub fn new<R>(channel: Channel, range: R) -> Self
+    where
+        R: Into<SeriesRange>,
+    {
+        Self {
+            channel,
+            range: range.into(),
+            one_before_range: false,
+            transform: TransformQuery::default_events(),
+            timeout: Some(Duration::from_millis(4000)),
+            events_max: Some(10000),
+            event_delay: None,
+            stream_batch_len: None,
+            buf_len_disk_io: None,
+            do_test_main_error: false,
+            do_test_stream_error: false,
+        }
+    }
+
+    pub fn channel(&self) -> &Channel {
+        &self.channel
+    }
+
+    pub fn range(&self) -> &SeriesRange {
+        &self.range
+    }
+
+    pub fn one_before_range(&self) -> bool {
+        self.one_before_range
+    }
+
+    pub fn transform(&self) -> &TransformQuery {
+        &self.transform
+    }
+
+    pub fn buf_len_disk_io(&self) -> usize {
+        self.buf_len_disk_io.unwrap_or(1024 * 8)
+    }
+
+    pub fn timeout(&self) -> Duration {
+        self.timeout.unwrap_or(Duration::from_millis(10000))
+    }
+
+    pub fn events_max(&self) -> u64 {
+        self.events_max.unwrap_or(1024 * 512)
+    }
+
+    pub fn event_delay(&self) -> &Option<Duration> {
+        &self.event_delay
+    }
+
+    pub fn do_test_main_error(&self) -> bool {
+        self.do_test_main_error
+    }
+
+    pub fn do_test_stream_error(&self) -> bool {
+        self.do_test_stream_error
+    }
+
+    pub fn set_series_id(&mut self, series: u64) {
+        self.channel.series = Some(series);
+    }
+
+    pub fn set_do_test_main_error(&mut self, k: bool) {
+        self.do_test_main_error = k;
+    }
+
+    pub fn set_do_test_stream_error(&mut self, k: bool) {
+        self.do_test_stream_error = k;
+    }
+
+    pub fn for_event_blobs(mut self) -> Self {
+        self.transform = TransformQuery::for_event_blobs();
+        self
+    }
+
+    pub fn for_time_weighted_scalar(mut self) -> Self {
+        self.transform = TransformQuery::for_time_weighted_scalar();
+        self
+    }
+
+    pub fn is_event_blobs(&self) -> bool {
+        self.transform.is_event_blobs()
+    }
+}
+
+impl HasBackend for PlainEventsQuery {
+    fn backend(&self) -> &str {
+        &self.channel.backend
+    }
+}
+
+impl HasTimeout for PlainEventsQuery {
+    fn timeout(&self) -> Duration {
+        self.timeout()
+    }
+}
+
+impl FromUrl for PlainEventsQuery {
+    fn from_url(url: &Url) -> Result<Self, Error> {
+        let pairs = get_url_query_pairs(url);
+        Self::from_pairs(&pairs)
+    }
+
+    fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
+        let range = if let Ok(x) = TimeRangeQuery::from_pairs(pairs) {
+            SeriesRange::TimeRange(x.into())
+        } else if let Ok(x) = PulseRangeQuery::from_pairs(pairs) {
+            SeriesRange::PulseRange(x.into())
+        } else {
+            return Err(Error::with_msg_no_trace("no series range in url"));
+        };
+        let ret = Self {
+            channel: Channel::from_pairs(pairs)?,
+            range,
+            one_before_range: pairs.get("oneBeforeRange").map_or("false", |x| x.as_ref()) == "true",
+            transform: TransformQuery::from_pairs(pairs)?,
+            timeout: pairs
+                .get("timeout")
+                .map(|x| x.parse::<u64>().map(Duration::from_millis).ok())
+                .unwrap_or(None),
+            events_max: pairs
+                .get("eventsMax")
+                .map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
+            event_delay: pairs.get("eventDelay").map_or(Ok(None), |k| {
+                k.parse::<u64>().map(|x| Duration::from_millis(x)).map(|k| Some(k))
+            })?,
+            stream_batch_len: pairs
+                .get("streamBatchLen")
+                .map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
+            buf_len_disk_io: pairs
+                .get("bufLenDiskIo")
+                .map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
+            do_test_main_error: pairs
+                .get("doTestMainError")
+                .map_or("false", |k| k)
+                .parse()
+                .map_err(|e| Error::with_public_msg(format!("can not parse doTestMainError: {}", e)))?,
+            do_test_stream_error: pairs
+                .get("doTestStreamError")
+                .map_or("false", |k| k)
+                .parse()
+                .map_err(|e| Error::with_public_msg(format!("can not parse doTestStreamError: {}", e)))?,
+        };
+        Ok(ret)
+    }
+}
+
+impl AppendToUrl for PlainEventsQuery {
+    fn append_to_url(&self, url: &mut Url) {
+        match &self.range {
+            SeriesRange::TimeRange(k) => TimeRangeQuery::from(k).append_to_url(url),
+            SeriesRange::PulseRange(_) => todo!(),
+        }
+        self.channel.append_to_url(url);
+        {
+            let mut g = url.query_pairs_mut();
+            if self.one_before_range() {
+                g.append_pair("oneBeforeRange", "true");
+            }
+        }
+        self.transform.append_to_url(url);
+        let mut g = url.query_pairs_mut();
+        if let Some(x) = &self.timeout {
+            g.append_pair("timeout", &format!("{}", x.as_millis()));
+        }
+        if let Some(x) = self.events_max.as_ref() {
+            g.append_pair("eventsMax", &format!("{}", x));
+        }
+        if let Some(x) = self.event_delay.as_ref() {
+            g.append_pair("eventDelay", &format!("{:.0}", x.as_secs_f64() * 1e3));
+        }
+        if let Some(x) = self.stream_batch_len.as_ref() {
+            g.append_pair("streamBatchLen", &format!("{}", x));
+        }
+        if let Some(x) = self.buf_len_disk_io.as_ref() {
+            g.append_pair("bufLenDiskIo", &format!("{}", x));
+        }
+        if self.do_test_main_error {
+            g.append_pair("doTestMainError", "true");
+        }
+        if self.do_test_stream_error {
+            g.append_pair("doTestStreamError", "true");
+        }
+    }
+}

query/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod api4;
+pub mod transform;

query/src/transform.rs

@@ -0,0 +1,161 @@
+use err::Error;
+use netpod::get_url_query_pairs;
+use netpod::log::*;
+use netpod::AppendToUrl;
+use netpod::FromUrl;
+use serde::Deserialize;
+use serde::Serialize;
+use std::collections::BTreeMap;
+use url::Url;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum EventTransform {
+    EventBlobsVerbatim,
+    EventBlobsUncompressed,
+    ValueFull,
+    ArrayPick(usize),
+    // TODO should rename to scalar? dim0 will only stay a scalar.
+    MinMaxAvgDev,
+    PulseIdDiff,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum TimeBinningTransform {
+    None,
+    TimeWeighted,
+    Unweighted,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct TransformQuery {
+    event: EventTransform,
+    time_binning: TimeBinningTransform,
+}
+
+impl TransformQuery {
+    fn url_prefix() -> &'static str {
+        "transform"
+    }
+
+    pub fn default_events() -> Self {
+        Self {
+            event: EventTransform::ValueFull,
+            time_binning: TimeBinningTransform::None,
+        }
+    }
+
+    pub fn default_time_binned() -> Self {
+        Self {
+            event: EventTransform::MinMaxAvgDev,
+            time_binning: TimeBinningTransform::TimeWeighted,
+        }
+    }
+
+    pub fn is_default_events(&self) -> bool {
+        self == &Self::default_events()
+    }
+
+    pub fn is_default_time_binned(&self) -> bool {
+        self == &Self::default_time_binned()
+    }
+
+    pub fn for_event_blobs() -> Self {
+        Self {
+            event: EventTransform::EventBlobsVerbatim,
+            time_binning: TimeBinningTransform::None,
+        }
+    }
+
+    pub fn for_time_weighted_scalar() -> Self {
+        Self {
+            event: EventTransform::MinMaxAvgDev,
+            time_binning: TimeBinningTransform::TimeWeighted,
+        }
+    }
+
+    pub fn is_event_blobs(&self) -> bool {
+        match &self.event {
+            EventTransform::EventBlobsVerbatim => true,
+            EventTransform::EventBlobsUncompressed => {
+                error!("TODO decide on uncompressed event blobs");
+                panic!()
+            }
+            _ => false,
+        }
+    }
+}
+
+impl FromUrl for TransformQuery {
+    fn from_url(url: &Url) -> Result<Self, Error> {
+        let pairs = get_url_query_pairs(url);
+        Self::from_pairs(&pairs)
+    }
+
+    fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
+        let upre = Self::url_prefix();
+        let key = "binningScheme";
+        if let Some(s) = pairs.get(key) {
+            let ret = if s == "eventBlobs" {
+                TransformQuery {
+                    event: EventTransform::EventBlobsVerbatim,
+                    time_binning: TimeBinningTransform::None,
+                }
+            } else if s == "fullValue" {
+                TransformQuery {
+                    event: EventTransform::ValueFull,
+                    time_binning: TimeBinningTransform::None,
+                }
+            } else if s == "timeWeightedScalar" {
+                TransformQuery {
+                    event: EventTransform::MinMaxAvgDev,
+                    time_binning: TimeBinningTransform::TimeWeighted,
+                }
+            } else if s == "unweightedScalar" {
+                TransformQuery {
+                    event: EventTransform::EventBlobsVerbatim,
+                    time_binning: TimeBinningTransform::None,
+                }
+            } else if s == "binnedX" {
+                let _u: usize = pairs.get("binnedXcount").map_or("1", |k| k).parse()?;
+                warn!("TODO binnedXcount");
+                TransformQuery {
+                    event: EventTransform::MinMaxAvgDev,
+                    time_binning: TimeBinningTransform::None,
+                }
+            } else if s == "pulseIdDiff" {
+                TransformQuery {
+                    event: EventTransform::PulseIdDiff,
+                    time_binning: TimeBinningTransform::None,
+                }
+            } else {
+                return Err(Error::with_msg("can not extract binningScheme"));
+            };
+            Ok(ret)
+        } else {
+            // TODO add option to pick from array.
+            let _pick = pairs
+                .get(&format!("{}ArrayPick", upre))
+                .map(|x| match x.parse::<usize>() {
+                    Ok(n) => Some(n),
+                    Err(_) => None,
+                })
+                .unwrap_or(None);
+            let ret = TransformQuery {
+                event: EventTransform::EventBlobsVerbatim,
+                time_binning: TimeBinningTransform::None,
+            };
+            Ok(ret)
+        }
+    }
+}
+
+impl AppendToUrl for TransformQuery {
+    fn append_to_url(&self, url: &mut Url) {
+        warn!("TODO AppendToUrl for Transform");
+        let upre = Self::url_prefix();
+        let mut g = url.query_pairs_mut();
+        if let Some(x) = &Some(123) {
+            g.append_pair(&format!("{}ArrayPick", upre), &format!("{}", x));
+        }
+    }
+}