Deliver plain dim-0 and dim-1 events as json

disk/src/binned/dim1.rs

@@ -0,0 +1,497 @@
+use crate::agg::binnedt::{TimeBinnableType, TimeBinnableTypeAggregator};
+use crate::agg::enp::WaveEvents;
+use crate::agg::streams::{Appendable, Collectable, Collector, ToJsonBytes, ToJsonResult};
+use crate::agg::{Fits, FitsInside};
+use crate::binned::{
+    Bool, FilterFittingInside, IsoDateTime, NumOps, RangeOverlapInfo, ReadPbv, ReadableFromFile, TimeBins, WithLen,
+};
+use crate::Sitemty;
+use chrono::{TimeZone, Utc};
+use err::Error;
+use netpod::timeunits::SEC;
+use netpod::NanoRange;
+use num_traits::Zero;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::marker::PhantomData;
+use tokio::fs::File;
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct MinMaxAvgDim1Bins<NTY> {
+    pub ts1s: Vec<u64>,
+    pub ts2s: Vec<u64>,
+    pub counts: Vec<u64>,
+    pub mins: Vec<Option<Vec<NTY>>>,
+    pub maxs: Vec<Option<Vec<NTY>>>,
+    pub avgs: Vec<Option<Vec<f32>>>,
+}
+
+impl<NTY> fmt::Debug for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: fmt::Debug,
+{
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            fmt,
+            "MinMaxAvgDim1Bins  count {}  ts1s {:?}  ts2s {:?}  counts {:?}  mins {:?}  maxs {:?}  avgs {:?}",
+            self.ts1s.len(),
+            self.ts1s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
+            self.ts2s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
+            self.counts,
+            self.mins.first(),
+            self.maxs.first(),
+            self.avgs.first(),
+        )
+    }
+}
+
+impl<NTY> MinMaxAvgDim1Bins<NTY> {
+    pub fn empty() -> Self {
+        Self {
+            ts1s: vec![],
+            ts2s: vec![],
+            counts: vec![],
+            mins: vec![],
+            maxs: vec![],
+            avgs: vec![],
+        }
+    }
+}
+
+impl<NTY> FitsInside for MinMaxAvgDim1Bins<NTY> {
+    fn fits_inside(&self, range: NanoRange) -> Fits {
+        if self.ts1s.is_empty() {
+            Fits::Empty
+        } else {
+            let t1 = *self.ts1s.first().unwrap();
+            let t2 = *self.ts2s.last().unwrap();
+            if t2 <= range.beg {
+                Fits::Lower
+            } else if t1 >= range.end {
+                Fits::Greater
+            } else if t1 < range.beg && t2 > range.end {
+                Fits::PartlyLowerAndGreater
+            } else if t1 < range.beg {
+                Fits::PartlyLower
+            } else if t2 > range.end {
+                Fits::PartlyGreater
+            } else {
+                Fits::Inside
+            }
+        }
+    }
+}
+
+impl<NTY> FilterFittingInside for MinMaxAvgDim1Bins<NTY> {
+    fn filter_fitting_inside(self, fit_range: NanoRange) -> Option<Self> {
+        match self.fits_inside(fit_range) {
+            Fits::Inside | Fits::PartlyGreater | Fits::PartlyLower | Fits::PartlyLowerAndGreater => Some(self),
+            _ => None,
+        }
+    }
+}
+
+impl<NTY> RangeOverlapInfo for MinMaxAvgDim1Bins<NTY> {
+    fn ends_before(&self, range: NanoRange) -> bool {
+        match self.ts2s.last() {
+            Some(&ts) => ts <= range.beg,
+            None => true,
+        }
+    }
+
+    fn ends_after(&self, range: NanoRange) -> bool {
+        match self.ts2s.last() {
+            Some(&ts) => ts > range.end,
+            None => panic!(),
+        }
+    }
+
+    fn starts_after(&self, range: NanoRange) -> bool {
+        match self.ts1s.first() {
+            Some(&ts) => ts >= range.end,
+            None => panic!(),
+        }
+    }
+}
+
+impl<NTY> TimeBins for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: NumOps,
+{
+    fn ts1s(&self) -> &Vec<u64> {
+        &self.ts1s
+    }
+
+    fn ts2s(&self) -> &Vec<u64> {
+        &self.ts2s
+    }
+}
+
+impl<NTY> WithLen for MinMaxAvgDim1Bins<NTY> {
+    fn len(&self) -> usize {
+        self.ts1s.len()
+    }
+}
+
+impl<NTY> Appendable for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: NumOps,
+{
+    fn empty() -> Self {
+        Self::empty()
+    }
+
+    fn append(&mut self, src: &Self) {
+        self.ts1s.extend_from_slice(&src.ts1s);
+        self.ts2s.extend_from_slice(&src.ts2s);
+        self.counts.extend_from_slice(&src.counts);
+        self.mins.extend_from_slice(&src.mins);
+        self.maxs.extend_from_slice(&src.maxs);
+        self.avgs.extend_from_slice(&src.avgs);
+    }
+}
+
+impl<NTY> ReadableFromFile for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: NumOps,
+{
+    // TODO this function is not needed in the trait:
+    fn read_from_file(file: File) -> Result<ReadPbv<Self>, Error> {
+        Ok(ReadPbv::new(file))
+    }
+
+    fn from_buf(buf: &[u8]) -> Result<Self, Error> {
+        let dec = serde_cbor::from_slice(&buf)?;
+        Ok(dec)
+    }
+}
+
+impl<NTY> TimeBinnableType for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: NumOps,
+{
+    type Output = MinMaxAvgDim1Bins<NTY>;
+    type Aggregator = MinMaxAvgDim1BinsAggregator<NTY>;
+
+    fn aggregator(range: NanoRange) -> Self::Aggregator {
+        Self::Aggregator::new(range)
+    }
+}
+
+impl<NTY> ToJsonResult for Sitemty<MinMaxAvgDim1Bins<NTY>>
+where
+    NTY: NumOps,
+{
+    fn to_json_result(&self) -> Result<Box<dyn ToJsonBytes>, Error> {
+        Ok(Box::new(serde_json::Value::String(format!(
+            "MinMaxAvgDim1Bins/non-json-item"
+        ))))
+    }
+}
+
+pub struct MinMaxAvgDim1BinsCollected<NTY> {
+    _m1: PhantomData<NTY>,
+}
+
+impl<NTY> MinMaxAvgDim1BinsCollected<NTY> {
+    pub fn new() -> Self {
+        Self { _m1: PhantomData }
+    }
+}
+
+#[derive(Serialize)]
+pub struct MinMaxAvgDim1BinsCollectedResult<NTY> {
+    ts_bin_edges: Vec<IsoDateTime>,
+    counts: Vec<u64>,
+    mins: Vec<Option<Vec<NTY>>>,
+    maxs: Vec<Option<Vec<NTY>>>,
+    avgs: Vec<Option<Vec<f32>>>,
+    #[serde(skip_serializing_if = "Bool::is_false", rename = "finalisedRange")]
+    finalised_range: bool,
+    #[serde(skip_serializing_if = "Zero::is_zero", rename = "missingBins")]
+    missing_bins: u32,
+    #[serde(skip_serializing_if = "Option::is_none", rename = "continueAt")]
+    continue_at: Option<IsoDateTime>,
+}
+
+pub struct MinMaxAvgDim1BinsCollector<NTY> {
+    bin_count_exp: u32,
+    timed_out: bool,
+    range_complete: bool,
+    vals: MinMaxAvgDim1Bins<NTY>,
+    _m1: PhantomData<NTY>,
+}
+
+impl<NTY> MinMaxAvgDim1BinsCollector<NTY> {
+    pub fn new(bin_count_exp: u32) -> Self {
+        Self {
+            bin_count_exp,
+            timed_out: false,
+            range_complete: false,
+            vals: MinMaxAvgDim1Bins::<NTY>::empty(),
+            _m1: PhantomData,
+        }
+    }
+}
+
+impl<NTY> WithLen for MinMaxAvgDim1BinsCollector<NTY>
+where
+    NTY: NumOps + Serialize,
+{
+    fn len(&self) -> usize {
+        self.vals.ts1s.len()
+    }
+}
+
+impl<NTY> Collector for MinMaxAvgDim1BinsCollector<NTY>
+where
+    NTY: NumOps + Serialize,
+{
+    type Input = MinMaxAvgDim1Bins<NTY>;
+    type Output = MinMaxAvgDim1BinsCollectedResult<NTY>;
+
+    fn ingest(&mut self, src: &Self::Input) {
+        Appendable::append(&mut self.vals, src);
+    }
+
+    fn set_range_complete(&mut self) {
+        self.range_complete = true;
+    }
+
+    fn set_timed_out(&mut self) {
+        self.timed_out = true;
+    }
+
+    fn result(self) -> Result<Self::Output, Error> {
+        let bin_count = self.vals.ts1s.len() as u32;
+        let mut tsa: Vec<_> = self
+            .vals
+            .ts1s
+            .iter()
+            .map(|&k| IsoDateTime(Utc.timestamp_nanos(k as i64)))
+            .collect();
+        if let Some(&z) = self.vals.ts2s.last() {
+            tsa.push(IsoDateTime(Utc.timestamp_nanos(z as i64)));
+        }
+        let tsa = tsa;
+        let continue_at = if self.vals.ts1s.len() < self.bin_count_exp as usize {
+            match tsa.last() {
+                Some(k) => Some(k.clone()),
+                None => Err(Error::with_msg("partial_content but no bin in result"))?,
+            }
+        } else {
+            None
+        };
+        let ret = MinMaxAvgDim1BinsCollectedResult::<NTY> {
+            ts_bin_edges: tsa,
+            counts: self.vals.counts,
+            mins: self.vals.mins,
+            maxs: self.vals.maxs,
+            avgs: self.vals.avgs,
+            finalised_range: self.range_complete,
+            missing_bins: self.bin_count_exp - bin_count,
+            continue_at,
+        };
+        Ok(ret)
+    }
+}
+
+impl<NTY> Collectable for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: NumOps + Serialize,
+{
+    type Collector = MinMaxAvgDim1BinsCollector<NTY>;
+
+    fn new_collector(bin_count_exp: u32) -> Self::Collector {
+        Self::Collector::new(bin_count_exp)
+    }
+}
+
+pub struct MinMaxAvgDim1BinsAggregator<NTY> {
+    range: NanoRange,
+    count: u64,
+    min: Option<Vec<NTY>>,
+    max: Option<Vec<NTY>>,
+    sumc: u64,
+    sum: Option<Vec<f32>>,
+}
+
+impl<NTY> MinMaxAvgDim1BinsAggregator<NTY> {
+    pub fn new(range: NanoRange) -> Self {
+        Self {
+            range,
+            count: 0,
+            min: None,
+            max: None,
+            sumc: 0,
+            sum: None,
+        }
+    }
+}
+
+impl<NTY> TimeBinnableTypeAggregator for MinMaxAvgDim1BinsAggregator<NTY>
+where
+    NTY: NumOps,
+{
+    type Input = MinMaxAvgDim1Bins<NTY>;
+    type Output = MinMaxAvgDim1Bins<NTY>;
+
+    fn range(&self) -> &NanoRange {
+        &self.range
+    }
+
+    fn ingest(&mut self, item: &Self::Input) {
+        for i1 in 0..item.ts1s.len() {
+            if item.ts2s[i1] <= self.range.beg {
+                continue;
+            } else if item.ts1s[i1] >= self.range.end {
+                continue;
+            } else {
+                match self.min.as_mut() {
+                    None => self.min = item.mins[i1].clone(),
+                    Some(min) => match item.mins[i1].as_ref() {
+                        None => {}
+                        Some(v) => {
+                            for (a, b) in min.iter_mut().zip(v.iter()) {
+                                if *b < *a {
+                                    *a = *b;
+                                }
+                            }
+                        }
+                    },
+                };
+                match self.max.as_mut() {
+                    None => self.max = item.maxs[i1].clone(),
+                    Some(max) => match item.maxs[i1].as_ref() {
+                        None => {}
+                        Some(v) => {
+                            for (a, b) in max.iter_mut().zip(v.iter()) {
+                                if *b > *a {
+                                    *a = *b;
+                                }
+                            }
+                        }
+                    },
+                };
+                match self.sum.as_mut() {
+                    None => {
+                        self.sum = item.avgs[i1].clone();
+                    }
+                    Some(sum) => match item.avgs[i1].as_ref() {
+                        None => {}
+                        Some(v) => {
+                            for (a, b) in sum.iter_mut().zip(v.iter()) {
+                                if (*b).is_nan() {
+                                } else {
+                                    *a += *b;
+                                }
+                            }
+                            self.sumc += 1;
+                        }
+                    },
+                }
+                self.count += item.counts[i1];
+            }
+        }
+    }
+
+    fn result(self) -> Self::Output {
+        let avg = if self.sumc == 0 {
+            None
+        } else {
+            let avg = self
+                .sum
+                .as_ref()
+                .unwrap()
+                .iter()
+                .map(|k| k / self.sumc as f32)
+                .collect();
+            Some(avg)
+        };
+        Self::Output {
+            ts1s: vec![self.range.beg],
+            ts2s: vec![self.range.end],
+            counts: vec![self.count],
+            mins: vec![self.min],
+            maxs: vec![self.max],
+            avgs: vec![avg],
+        }
+    }
+}
+
+#[derive(Serialize)]
+pub struct WaveEventsCollectedResult<NTY> {
+    ts0: u64,
+    tsoff: Vec<u64>,
+    values: Vec<Vec<NTY>>,
+    #[serde(skip_serializing_if = "Bool::is_false", rename = "finalisedRange")]
+    range_complete: bool,
+    #[serde(skip_serializing_if = "Bool::is_false", rename = "timedOut")]
+    timed_out: bool,
+}
+
+pub struct WaveEventsCollector<NTY> {
+    vals: WaveEvents<NTY>,
+    range_complete: bool,
+    timed_out: bool,
+}
+
+impl<NTY> WaveEventsCollector<NTY> {
+    pub fn new(_bin_count_exp: u32) -> Self {
+        Self {
+            vals: WaveEvents::empty(),
+            range_complete: false,
+            timed_out: false,
+        }
+    }
+}
+
+impl<NTY> WithLen for WaveEventsCollector<NTY> {
+    fn len(&self) -> usize {
+        self.vals.tss.len()
+    }
+}
+
+impl<NTY> Collector for WaveEventsCollector<NTY>
+where
+    NTY: NumOps,
+{
+    type Input = WaveEvents<NTY>;
+    type Output = WaveEventsCollectedResult<NTY>;
+
+    fn ingest(&mut self, src: &Self::Input) {
+        self.vals.append(src);
+    }
+
+    fn set_range_complete(&mut self) {
+        self.range_complete = true;
+    }
+
+    fn set_timed_out(&mut self) {
+        self.timed_out = true;
+    }
+
+    fn result(self) -> Result<Self::Output, Error> {
+        let ts0 = self.vals.tss.first().map_or(0, |k| *k / SEC);
+        let tsoff = self.vals.tss.into_iter().map(|k| k - ts0 * SEC).collect();
+        let ret = Self::Output {
+            ts0,
+            tsoff,
+            values: self.vals.vals,
+            range_complete: self.range_complete,
+            timed_out: self.timed_out,
+        };
+        Ok(ret)
+    }
+}
+
+impl<NTY> Collectable for WaveEvents<NTY>
+where
+    NTY: NumOps,
+{
+    type Collector = WaveEventsCollector<NTY>;
+
+    fn new_collector(bin_count_exp: u32) -> Self::Collector {
+        Self::Collector::new(bin_count_exp)
+    }
+}

disk/src/binned/prebinned.rs

@@ -115,7 +115,18 @@ pub async fn pre_binned_bytes_for_http(
         ));
         return Err(err);
     }
-    let channel_config = read_local_config(&query.channel(), &node_config.node).await?;
+    let channel_config = match read_local_config(&query.channel(), &node_config.node).await {
+        Ok(k) => k,
+        Err(e) => {
+            if e.msg().contains("ErrorKind::NotFound") {
+                let s = futures_util::stream::empty();
+                let ret = Box::pin(s);
+                return Ok(ret);
+            } else {
+                return Err(e);
+            }
+        }
+    };
     let entry_res = extract_matching_config_entry(&query.patch().patch_range(), &channel_config)?;
     let entry = match entry_res {
         MatchingConfigEntry::None => return Err(Error::with_msg("no config entry found")),

disk/src/binned/query.rs

@@ -1,3 +1,4 @@
+use crate::query::channel_from_params;
 use chrono::{DateTime, TimeZone, Utc};
 use err::Error;
 use netpod::log::*;
@@ -211,7 +212,7 @@ impl BinnedQuery {
                 .map_err(|e| Error::with_msg(format!("can not parse binCount {:?}", e)))?,
             agg_kind: params
                 .get("aggKind")
-                .map_or("DimXBins1", |k| k)
+                .map_or(&format!("{}", AggKind::DimXBins1), |k| k)
                 .parse()
                 .map_err(|e| Error::with_msg(format!("can not parse aggKind {:?}", e)))?,
             channel: channel_from_params(&params)?,
@@ -306,96 +307,3 @@ impl BinnedQuery {
         )
     }
 }
-
-fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Error> {
-    let ret = Channel {
-        backend: params
-            .get("channelBackend")
-            .ok_or(Error::with_msg("missing channelBackend"))?
-            .into(),
-        name: params
-            .get("channelName")
-            .ok_or(Error::with_msg("missing channelName"))?
-            .into(),
-    };
-    Ok(ret)
-}
-
-// TODO move this query type out of this `binned` mod
-#[derive(Clone, Debug)]
-pub struct PlainEventsQuery {
-    channel: Channel,
-    range: NanoRange,
-    report_error: bool,
-    timeout: Duration,
-}
-
-impl PlainEventsQuery {
-    pub fn new(channel: Channel, range: NanoRange) -> Self {
-        Self {
-            channel,
-            range,
-            report_error: false,
-            timeout: Duration::from_millis(2000),
-        }
-    }
-
-    pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
-        let params = netpod::query_params(req.uri.query());
-        let beg_date = params.get("begDate").ok_or(Error::with_msg("missing begDate"))?;
-        let end_date = params.get("endDate").ok_or(Error::with_msg("missing endDate"))?;
-        let ret = Self {
-            range: NanoRange {
-                beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
-                end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
-            },
-            channel: channel_from_params(&params)?,
-            report_error: params
-                .get("reportError")
-                .map_or("false", |k| k)
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,
-            timeout: params
-                .get("timeout")
-                .map_or("2000", |k| k)
-                .parse::<u64>()
-                .map(|k| Duration::from_millis(k))
-                .map_err(|e| Error::with_msg(format!("can not parse timeout {:?}", e)))?,
-        };
-        Ok(ret)
-    }
-
-    pub fn range(&self) -> &NanoRange {
-        &self.range
-    }
-
-    pub fn channel(&self) -> &Channel {
-        &self.channel
-    }
-
-    pub fn report_error(&self) -> bool {
-        self.report_error
-    }
-
-    pub fn timeout(&self) -> Duration {
-        self.timeout
-    }
-
-    pub fn set_timeout(&mut self, k: Duration) {
-        self.timeout = k;
-    }
-
-    pub fn url(&self, host: &HostPort) -> String {
-        let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
-        format!(
-            "http://{}:{}/api/4/plain_events_json?channelBackend={}&channelName={}&begDate={}&endDate={}&timeout={}",
-            host.host,
-            host.port,
-            self.channel.backend,
-            self.channel.name,
-            Utc.timestamp_nanos(self.range.beg as i64).format(date_fmt),
-            Utc.timestamp_nanos(self.range.end as i64).format(date_fmt),
-            self.timeout.as_millis(),
-        )
-    }
-}