Refactor and prepare for scylla based bin caching

This commit is contained in:
Dominik Werder
2022-07-06 15:51:05 +02:00
parent 66215f583f
commit d0a7240934
29 changed files with 1647 additions and 542 deletions
+10 -2
View File
@@ -42,13 +42,21 @@ where
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output: SitemtyFrameType + TimeBinned,
{
if let Some(scyconf) = &node_config.node_config.cluster.cache_scylla {
info!("~~~~~~~~~~~~~~~ make_num_pipeline_nty_end_evs_enp using scylla as cache");
trace!("~~~~~~~~~~~~~~~ make_num_pipeline_nty_end_evs_enp using scylla as cache");
let chn = ChannelTyped {
channel: query.channel().clone(),
scalar_type,
shape,
};
let stream = pre_binned_value_stream(&chn, query.patch(), scyconf).await?;
let stream = pre_binned_value_stream(
chn.channel().series().unwrap(),
&chn,
query.patch(),
agg_kind,
query.cache_usage(),
scyconf,
)
.await?;
let stream = stream.map(|x| {
//
match x {
+3 -6
View File
@@ -2,10 +2,7 @@ use err::Error;
use http::request::Parts;
use netpod::query::{agg_kind_from_binning_scheme, binning_scheme_append_to_url, CacheUsage};
use netpod::timeunits::SEC;
use netpod::{
channel_append_to_url, channel_from_pairs, AggKind, AppendToUrl, ByteSize, Channel, PreBinnedPatchCoord,
ScalarType, Shape,
};
use netpod::{AggKind, AppendToUrl, ByteSize, Channel, FromUrl, PreBinnedPatchCoord, ScalarType, Shape};
use std::collections::BTreeMap;
use url::Url;
@@ -81,7 +78,7 @@ impl PreBinnedQuery {
.map(|x| Shape::from_url_str(&x))??;
let ret = Self {
patch: PreBinnedPatchCoord::new(bin_t_len * SEC, patch_t_len * SEC, patch_ix),
channel: channel_from_pairs(&pairs)?,
channel: Channel::from_pairs(&pairs)?,
scalar_type,
shape,
agg_kind: agg_kind_from_binning_scheme(&pairs).unwrap_or(AggKind::DimXBins1),
@@ -148,7 +145,7 @@ impl AppendToUrl for PreBinnedQuery {
fn append_to_url(&self, url: &mut Url) {
self.patch.append_to_url(url);
binning_scheme_append_to_url(&self.agg_kind, url);
channel_append_to_url(url, &self.channel);
self.channel.append_to_url(url);
self.shape.append_to_url(url);
self.scalar_type.append_to_url(url);
let mut g = url.query_pairs_mut();