daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove BinSpecDimT in favor of BinnedRange

Browse Source
This commit is contained in:
Dominik Werder
2021-04-29 09:50:17 +02:00
parent de952a6e64
commit 3b2059f76d
5 changed files with 4 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
disk/src/agg/binnedt.rs
View File

@@ -3,7 +3,7 @@ use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{BinSpecDimT, BinnedRange};
use netpod::BinnedRange;
use std::pin::Pin;
use std::task::{Context, Poll};

2
disk/src/aggtest.rs
View File

@@ -5,7 +5,7 @@ use crate::agg::binnedx::IntoBinnedXBins1;
use crate::agg::make_test_node;
use futures_util::StreamExt;
use netpod::timeunits::*;
use netpod::{BinSpecDimT, BinnedRange, Channel, ChannelConfig, NanoRange, Nanos, ScalarType, Shape};
use netpod::{BinnedRange, Channel, ChannelConfig, NanoRange, Nanos, ScalarType, Shape};
use std::future::ready;
#[allow(unused_imports)]
use tracing::{debug, error, info, trace, warn};

3
disk/src/binnedstream.rs
View File

@@ -5,8 +5,7 @@ use futures_core::Stream;
use futures_util::StreamExt;
#[allow(unused_imports)]
use netpod::log::*;
use netpod::{AggKind, BinSpecDimT, BinnedRange, Channel, NodeConfig, PreBinnedPatchIterator};
use netpod::{NanoRange, RetStreamExt};
use netpod::{AggKind, BinnedRange, Channel, NodeConfig, PreBinnedPatchIterator};
use std::future::ready;
use std::pin::Pin;
use std::task::{Context, Poll};

11
disk/src/cache/pbv.rs vendored
View File

@@ -10,7 +10,7 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt, TryStreamExt};
use netpod::log::*;
use netpod::{
AggKind, BinSpecDimT, BinnedRange, Channel, NanoRange, NodeConfig, PreBinnedPatchCoord, PreBinnedPatchIterator,
AggKind, BinnedRange, Channel, NanoRange, NodeConfig, PreBinnedPatchCoord, PreBinnedPatchIterator,
PreBinnedPatchRange,
};
use std::future::{ready, Future};
@@ -132,15 +132,6 @@ impl PreBinnedValueStream {
assert!(self.patch_coord.patch_t_len() % self.patch_coord.bin_t_len() == 0);
error!("try_setup_fetch_prebinned_higher_res apply all requested transformations and T-binning");
let count = self.patch_coord.patch_t_len() / self.patch_coord.bin_t_len();
// TODO use a ctor, remove from BinSpecDimT the redundant variable.
// If given a timestamp range, verify that it divides.
// For ranges, use a range type.
let _spec = BinSpecDimT {
bs: self.patch_coord.bin_t_len(),
ts1: self.patch_coord.patch_beg(),
ts2: self.patch_coord.patch_end(),
count,
};
let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap();
let s1 = MergedFromRemotes::new(evq, self.node_config.cluster.clone());
let s2 = s1

49
netpod/src/lib.rs
View File

@@ -3,7 +3,6 @@ use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -217,54 +216,6 @@ pub mod timeunits {
pub const WEEK: u64 = DAY * 7;
}
pub struct BinSpecDimT {
pub count: u64,
pub ts1: u64,
pub ts2: u64,
pub bs: u64,
}
impl BinSpecDimT {
pub fn over_range(count: u64, ts1: u64, ts2: u64) -> Self {
use timeunits::*;
assert!(count >= 1);
assert!(count <= 2000);
assert!(ts2 > ts1);
let dt = ts2 - ts1;
assert!(dt <= DAY * 14);
let bs = dt / count;
let mut i1 = 0;
let bs = loop {
if i1 >= BIN_THRESHOLDS.len() {
break *BIN_THRESHOLDS.last().unwrap();
}
let t = BIN_THRESHOLDS[i1];
if bs <= t {
break t;
}
i1 += 1;
};
//info!("INPUT TS {} {}", ts1, ts2);
//info!("chosen binsize: {} {}", i1, bs);
let ts1 = ts1 / bs * bs;
let ts2 = (ts2 + bs - 1) / bs * bs;
//info!("ADJUSTED TS {} {}", ts1, ts2);
BinSpecDimT {
count: (ts2 - ts1) / bs,
ts1,
ts2,
bs,
}
}
pub fn get_range(&self, ix: u32) -> NanoRange {
NanoRange {
beg: self.ts1 + ix as u64 * self.bs,
end: self.ts1 + (ix as u64 + 1) * self.bs,
}
}
}
const BIN_T_LEN_OPTIONS: [u64; 6] = [SEC * 10, MIN * 10, HOUR, HOUR * 4, DAY, DAY * 4];
const PATCH_T_LEN_OPTIONS: [u64; 6] = [MIN * 10, HOUR, HOUR * 4, DAY, DAY * 4, DAY * 12];