Move query types to module

disk/src/cache.rs

@@ -1,23 +1,18 @@
 use crate::agg::streams::StreamItem;
 use crate::binned::{RangeCompletableItem, StreamKind};
-use crate::cache::pbv::PreBinnedValueByteStream;
-use crate::frame::makeframe::FrameType;
 use crate::merge::MergedStream;
 use crate::raw::EventsQuery;
 use bytes::Bytes;
-use chrono::{DateTime, TimeZone, Utc};
+use chrono::Utc;
 use err::Error;
 use futures_core::Stream;
 use futures_util::{pin_mut, StreamExt};
 use hyper::{Body, Response};
 use netpod::log::*;
 use netpod::timeunits::SEC;
-use netpod::{
-    AggKind, ByteSize, Channel, Cluster, HostPort, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchCoord, ToNanos,
-};
+use netpod::{AggKind, Channel, Cluster, NodeConfigCached, PerfOpts, PreBinnedPatchCoord};
 use serde::{Deserialize, Serialize};
-use std::collections::{BTreeMap, VecDeque};
-use std::fmt::{Display, Formatter};
+use std::collections::VecDeque;
 use std::future::Future;
 use std::io;
 use std::path::PathBuf;
@@ -29,308 +24,6 @@ use tokio::io::{AsyncRead, ReadBuf};
 pub mod pbv;
 pub mod pbvfs;
 
-#[derive(Clone, Debug)]
-pub enum CacheUsage {
-    Use,
-    Ignore,
-    Recreate,
-}
-
-impl CacheUsage {
-    pub fn query_param_value(&self) -> String {
-        match self {
-            CacheUsage::Use => "use",
-            CacheUsage::Ignore => "ignore",
-            CacheUsage::Recreate => "recreate",
-        }
-        .into()
-    }
-
-    pub fn from_params(params: &BTreeMap<String, String>) -> Result<Self, Error> {
-        let ret = params.get("cacheUsage").map_or(Ok::<_, Error>(CacheUsage::Use), |k| {
-            if k == "use" {
-                Ok(CacheUsage::Use)
-            } else if k == "ignore" {
-                Ok(CacheUsage::Ignore)
-            } else if k == "recreate" {
-                Ok(CacheUsage::Recreate)
-            } else {
-                Err(Error::with_msg(format!("unexpected cacheUsage {:?}", k)))?
-            }
-        })?;
-        Ok(ret)
-    }
-
-    pub fn from_string(s: &str) -> Result<Self, Error> {
-        let ret = if s == "ignore" {
-            CacheUsage::Ignore
-        } else if s == "recreate" {
-            CacheUsage::Recreate
-        } else if s == "use" {
-            CacheUsage::Use
-        } else {
-            return Err(Error::with_msg(format!("can not interpret cache usage string: {}", s)));
-        };
-        Ok(ret)
-    }
-}
-
-impl Display for CacheUsage {
-    fn fmt(&self, fmt: &mut Formatter) -> std::fmt::Result {
-        write!(fmt, "{}", self.query_param_value())
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct BinnedQuery {
-    channel: Channel,
-    range: NanoRange,
-    bin_count: u32,
-    agg_kind: AggKind,
-    cache_usage: CacheUsage,
-    disk_stats_every: ByteSize,
-    report_error: bool,
-}
-
-impl BinnedQuery {
-    pub fn new(channel: Channel, range: NanoRange, bin_count: u32, agg_kind: AggKind) -> BinnedQuery {
-        BinnedQuery {
-            channel,
-            range,
-            bin_count,
-            agg_kind,
-            cache_usage: CacheUsage::Use,
-            disk_stats_every: ByteSize(1024 * 1024 * 4),
-            report_error: false,
-        }
-    }
-
-    pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
-        let params = netpod::query_params(req.uri.query());
-        let beg_date = params.get("begDate").ok_or(Error::with_msg("missing begDate"))?;
-        let end_date = params.get("endDate").ok_or(Error::with_msg("missing endDate"))?;
-        let disk_stats_every = params.get("diskStatsEveryKb").map_or("2000", |k| k);
-        let disk_stats_every = disk_stats_every
-            .parse()
-            .map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
-        let ret = BinnedQuery {
-            range: NanoRange {
-                beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
-                end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
-            },
-            bin_count: params
-                .get("binCount")
-                .ok_or(Error::with_msg("missing binCount"))?
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse binCount {:?}", e)))?,
-            agg_kind: params
-                .get("aggKind")
-                .map_or("DimXBins1", |k| k)
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse aggKind {:?}", e)))?,
-            channel: channel_from_params(&params)?,
-            cache_usage: CacheUsage::from_params(&params)?,
-            disk_stats_every: ByteSize::kb(disk_stats_every),
-            report_error: params
-                .get("reportError")
-                .map_or("false", |k| k)
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,
-        };
-        info!("BinnedQuery::from_request  {:?}", ret);
-        Ok(ret)
-    }
-
-    pub fn range(&self) -> &NanoRange {
-        &self.range
-    }
-
-    pub fn channel(&self) -> &Channel {
-        &self.channel
-    }
-
-    pub fn bin_count(&self) -> u32 {
-        self.bin_count
-    }
-
-    pub fn agg_kind(&self) -> &AggKind {
-        &self.agg_kind
-    }
-
-    pub fn cache_usage(&self) -> &CacheUsage {
-        &self.cache_usage
-    }
-
-    pub fn disk_stats_every(&self) -> &ByteSize {
-        &self.disk_stats_every
-    }
-
-    pub fn report_error(&self) -> bool {
-        self.report_error
-    }
-
-    pub fn set_cache_usage(&mut self, k: CacheUsage) {
-        self.cache_usage = k;
-    }
-
-    pub fn set_disk_stats_every(&mut self, k: ByteSize) {
-        self.disk_stats_every = k;
-    }
-
-    // TODO the BinnedQuery itself should maybe already carry the full HostPort?
-    // On the other hand, want to keep the flexibility for the fail over possibility..
-    pub fn url(&self, host: &HostPort) -> String {
-        let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
-        format!(
-            "http://{}:{}/api/4/binned?cacheUsage={}&channelBackend={}&channelName={}&binCount={}&begDate={}&endDate={}&diskStatsEveryKb={}",
-            host.host,
-            host.port,
-            self.cache_usage,
-            self.channel.backend,
-            self.channel.name,
-            self.bin_count,
-            Utc.timestamp_nanos(self.range.beg as i64).format(date_fmt),
-            Utc.timestamp_nanos(self.range.end as i64).format(date_fmt),
-            self.disk_stats_every.bytes() / 1024,
-        )
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct PreBinnedQuery {
-    patch: PreBinnedPatchCoord,
-    agg_kind: AggKind,
-    channel: Channel,
-    cache_usage: CacheUsage,
-    disk_stats_every: ByteSize,
-    report_error: bool,
-}
-
-impl PreBinnedQuery {
-    pub fn new(
-        patch: PreBinnedPatchCoord,
-        channel: Channel,
-        agg_kind: AggKind,
-        cache_usage: CacheUsage,
-        disk_stats_every: ByteSize,
-        report_error: bool,
-    ) -> Self {
-        Self {
-            patch,
-            agg_kind,
-            channel,
-            cache_usage,
-            disk_stats_every,
-            report_error,
-        }
-    }
-
-    pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
-        let params = netpod::query_params(req.uri.query());
-        let patch_ix = params
-            .get("patchIx")
-            .ok_or(Error::with_msg("missing patchIx"))?
-            .parse()?;
-        let bin_t_len = params
-            .get("binTlen")
-            .ok_or(Error::with_msg("missing binTlen"))?
-            .parse()?;
-        let patch_t_len = params
-            .get("patchTlen")
-            .ok_or(Error::with_msg("missing patchTlen"))?
-            .parse()?;
-        let disk_stats_every = params
-            .get("diskStatsEveryKb")
-            .ok_or(Error::with_msg("missing diskStatsEveryKb"))?;
-        let disk_stats_every = disk_stats_every
-            .parse()
-            .map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
-        let ret = PreBinnedQuery {
-            patch: PreBinnedPatchCoord::new(bin_t_len, patch_t_len, patch_ix),
-            agg_kind: params
-                .get("aggKind")
-                .map_or(&format!("{}", AggKind::DimXBins1), |k| k)
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse aggKind {:?}", e)))?,
-            channel: channel_from_params(&params)?,
-            cache_usage: CacheUsage::from_params(&params)?,
-            disk_stats_every: ByteSize::kb(disk_stats_every),
-            report_error: params
-                .get("reportError")
-                .map_or("false", |k| k)
-                .parse()
-                .map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,
-        };
-        Ok(ret)
-    }
-
-    pub fn make_query_string(&self) -> String {
-        format!(
-            "{}&channelBackend={}&channelName={}&aggKind={}&cacheUsage={}&diskStatsEveryKb={}&reportError={}",
-            self.patch.to_url_params_strings(),
-            self.channel.backend,
-            self.channel.name,
-            self.agg_kind,
-            self.cache_usage,
-            self.disk_stats_every.bytes() / 1024,
-            self.report_error(),
-        )
-    }
-
-    pub fn patch(&self) -> &PreBinnedPatchCoord {
-        &self.patch
-    }
-
-    pub fn report_error(&self) -> bool {
-        self.report_error
-    }
-}
-
-fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Error> {
-    let ret = Channel {
-        backend: params
-            .get("channelBackend")
-            .ok_or(Error::with_msg("missing channelBackend"))?
-            .into(),
-        name: params
-            .get("channelName")
-            .ok_or(Error::with_msg("missing channelName"))?
-            .into(),
-    };
-    Ok(ret)
-}
-
-// NOTE  This answers a request for a single valid pre-binned patch.
-// A user must first make sure that the grid spec is valid, and that this node is responsible for it.
-// Otherwise it is an error.
-pub fn pre_binned_bytes_for_http<SK>(
-    node_config: &NodeConfigCached,
-    query: &PreBinnedQuery,
-    stream_kind: SK,
-) -> Result<PreBinnedValueByteStream<SK>, Error>
-where
-    SK: StreamKind,
-    Result<StreamItem<RangeCompletableItem<SK::TBinnedBins>>, err::Error>: FrameType,
-{
-    if query.channel.backend != node_config.node.backend {
-        let err = Error::with_msg(format!(
-            "backend mismatch  node: {}  requested: {}",
-            node_config.node.backend, query.channel.backend
-        ));
-        return Err(err);
-    }
-    let patch_node_ix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
-    if node_config.ix as u32 != patch_node_ix {
-        Err(Error::with_msg(format!(
-            "pre_binned_bytes_for_http node mismatch  node_config.ix {}  patch_node_ix {}",
-            node_config.ix, patch_node_ix
-        )))
-    } else {
-        let ret = crate::cache::pbv::pre_binned_value_byte_stream_new(query, node_config, stream_kind);
-        Ok(ret)
-    }
-}
-
 pub struct HttpBodyAsAsyncRead {
     inp: Response<Body>,
     left: Bytes,