daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-09-18 12:12:53 +02:00
parent ab6b0322c9
commit e4f8ad1e91
25 changed files with 520 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/dbconn/src/dbconn.rs
View File

@@ -65,7 +65,7 @@ pub async fn delay_io_medium() {
}
pub async fn create_connection(db_config: &Database) -> Result<(PgClient, JoinHandle<Result<(), Error>>), Error> {
warn!("create_connection\n\n CREATING POSTGRES CONNECTION\n\n");
warn!("create_connection creating postgres connection");
// TODO use a common already running worker pool for these queries:
let d = db_config;
let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name);

13
crates/items_0/src/scalar_ops.rs
View File

@@ -64,7 +64,18 @@ impl AsPrimF32 for String {
}
pub trait ScalarOps:
fmt::Debug + Clone + PartialOrd + PartialEq + SubFrId + AsPrimF32 + ByteEstimate + Serialize + Unpin + Send + 'static
fmt::Debug
+ fmt::Display
+ Clone
+ PartialOrd
+ PartialEq
+ SubFrId
+ AsPrimF32
+ ByteEstimate
+ Serialize
+ Unpin
+ Send
+ 'static
{
fn scalar_type_name() -> &'static str;
fn zero_b() -> Self;

7
crates/items_0/src/timebin.rs
View File

@@ -191,18 +191,17 @@ impl TimeBinnable for Box<dyn TimeBinnable> {
}
}
#[allow(unused)]
impl RangeOverlapInfo for Box<dyn Events> {
fn ends_before(&self, range: &SeriesRange) -> bool {
todo!()
RangeOverlapInfo::ends_before(self.as_ref(), range)
}
fn ends_after(&self, range: &SeriesRange) -> bool {
todo!()
RangeOverlapInfo::ends_after(self.as_ref(), range)
}
fn starts_after(&self, range: &SeriesRange) -> bool {
todo!()
RangeOverlapInfo::starts_after(self.as_ref(), range)
}
}

270
crates/items_2/src/binsdim0.rs
View File

@@ -52,7 +52,7 @@ use std::mem;
use std::ops::Range;
#[allow(unused)]
macro_rules! trace4 {
macro_rules! trace44 {
($($arg:tt)*) => ();
($($arg:tt)*) => (eprintln!($($arg)*));
}
@@ -62,10 +62,11 @@ macro_rules! trace4 {
pub struct BinsDim0<NTY> {
pub ts1s: VecDeque<u64>,
pub ts2s: VecDeque<u64>,
pub counts: VecDeque<u64>,
pub cnts: VecDeque<u64>,
pub mins: VecDeque<NTY>,
pub maxs: VecDeque<NTY>,
pub avgs: VecDeque<f32>,
pub lsts: VecDeque<NTY>,
pub dim0kind: Option<Dim0Kind>,
}
@@ -88,7 +89,7 @@ where
self.ts1s.len(),
self.ts1s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.ts2s.iter().map(|k| k / SEC).collect::<Vec<_>>(),
self.counts,
self.cnts,
self.mins,
self.maxs,
self.avgs,
@@ -100,8 +101,8 @@ where
self.ts1s.len(),
self.ts1s.front().map(|k| k / SEC),
self.ts2s.back().map(|k| k / SEC),
self.counts.front(),
self.counts.back(),
self.cnts.front(),
self.cnts.back(),
self.avgs.front(),
self.avgs.back(),
)
@@ -109,48 +110,80 @@ where
}
}
trait HasFrontBack<T> {
fn len(&self) -> usize;
fn front(&self) -> Option<&T>;
fn back(&self) -> Option<&T>;
}
impl<T> HasFrontBack<T> for VecDeque<T> {
fn len(&self) -> usize {
self.len()
}
fn front(&self) -> Option<&T> {
self.front()
}
fn back(&self) -> Option<&T> {
self.back()
}
}
struct VecPreview<'a, T> {
c: &'a dyn HasFrontBack<T>,
}
impl<'a, T> VecPreview<'a, T> {
fn new(c: &'a dyn HasFrontBack<T>) -> Self {
Self { c }
}
}
impl<'a, T> fmt::Display for VecPreview<'a, T>
where
T: fmt::Display,
{
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
if self.c.len() == 0 {
write!(fmt, "()")
} else if self.c.len() == 1 {
write!(fmt, "{}", self.c.front().unwrap())
} else {
write!(fmt, "{}", self.c.front().unwrap())
}
}
}
impl<NTY> fmt::Display for BinsDim0<NTY>
where
NTY: fmt::Debug,
NTY: fmt::Display,
{
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
let self_name = any::type_name::<Self>();
if true {
write!(
fmt,
"{self_name} count {:?} ts1s {:?} ts2s {:?} counts {:?} mins {:?} maxs {:?} avgs {:?}",
self.ts1s.len(),
self.ts1s.iter().map(|&k| TsNano::from_ns(k)).collect::<Vec<_>>(),
self.ts2s.iter().map(|&k| TsNano::from_ns(k)).collect::<Vec<_>>(),
self.counts,
self.mins,
self.maxs,
self.avgs,
)
} else {
write!(
fmt,
"{self_name} count {:?} edges {:?} .. {:?} counts {:?} .. {:?} avgs {:?} .. {:?}",
self.ts1s.len(),
self.ts1s.front().map(|&k| TsNano::from_ns(k)),
self.ts2s.back().map(|&k| TsNano::from_ns(k)),
self.counts.front(),
self.counts.back(),
self.avgs.front(),
self.avgs.back(),
)
}
write!(
fmt,
"{self_name} {{ len: {:?}, ts1s: {}, ts2s {}, counts {}, mins {}, maxs {}, avgs {} }}",
self.len(),
VecPreview::new(&self.ts1s),
VecPreview::new(&self.ts2s),
VecPreview::new(&self.cnts),
VecPreview::new(&self.mins),
VecPreview::new(&self.maxs),
VecPreview::new(&self.avgs),
)
}
}
impl<NTY: ScalarOps> BinsDim0<NTY> {
pub fn push(&mut self, ts1: u64, ts2: u64, count: u64, min: NTY, max: NTY, avg: f32) {
pub fn push(&mut self, ts1: u64, ts2: u64, count: u64, min: NTY, max: NTY, avg: f32, lst: NTY) {
self.ts1s.push_back(ts1);
self.ts2s.push_back(ts2);
self.counts.push_back(count);
self.cnts.push_back(count);
self.mins.push_back(min);
self.maxs.push_back(max);
self.avgs.push_back(avg);
self.lsts.push_back(lst);
}
pub fn equal_slack(&self, other: &Self) -> bool {
@@ -190,10 +223,11 @@ impl<NTY: ScalarOps> BinsDim0<NTY> {
pub fn drain_into(&mut self, dst: &mut Self, range: Range<usize>) -> () {
dst.ts1s.extend(self.ts1s.drain(range.clone()));
dst.ts2s.extend(self.ts2s.drain(range.clone()));
dst.counts.extend(self.counts.drain(range.clone()));
dst.cnts.extend(self.cnts.drain(range.clone()));
dst.mins.extend(self.mins.drain(range.clone()));
dst.maxs.extend(self.maxs.drain(range.clone()));
dst.avgs.extend(self.avgs.drain(range.clone()));
dst.lsts.extend(self.lsts.drain(range.clone()));
}
}
@@ -220,10 +254,11 @@ impl<STY> Empty for BinsDim0<STY> {
Self {
ts1s: VecDeque::new(),
ts2s: VecDeque::new(),
counts: VecDeque::new(),
cnts: VecDeque::new(),
mins: VecDeque::new(),
maxs: VecDeque::new(),
avgs: VecDeque::new(),
lsts: VecDeque::new(),
dim0kind: None,
}
}
@@ -257,7 +292,7 @@ impl<STY> Resettable for BinsDim0<STY> {
fn reset(&mut self) {
self.ts1s.clear();
self.ts2s.clear();
self.counts.clear();
self.cnts.clear();
self.mins.clear();
self.maxs.clear();
self.avgs.clear();
@@ -266,7 +301,7 @@ impl<STY> Resettable for BinsDim0<STY> {
impl<STY: ScalarOps> HasNonemptyFirstBin for BinsDim0<STY> {
fn has_nonempty_first_bin(&self) -> bool {
self.counts.front().map_or(false, |x| *x > 0)
self.cnts.front().map_or(false, |x| *x > 0)
}
}
@@ -294,7 +329,7 @@ impl<NTY: ScalarOps> AppendEmptyBin for BinsDim0<NTY> {
fn append_empty_bin(&mut self, ts1: u64, ts2: u64) {
self.ts1s.push_back(ts1);
self.ts2s.push_back(ts2);
self.counts.push_back(0);
self.cnts.push_back(0);
self.mins.push_back(NTY::zero_b());
self.maxs.push_back(NTY::zero_b());
self.avgs.push_back(0.);
@@ -305,7 +340,7 @@ impl<NTY: ScalarOps> AppendAllFrom for BinsDim0<NTY> {
fn append_all_from(&mut self, src: &mut Self) {
self.ts1s.extend(src.ts1s.drain(..));
self.ts2s.extend(src.ts2s.drain(..));
self.counts.extend(src.counts.drain(..));
self.cnts.extend(src.cnts.drain(..));
self.mins.extend(src.mins.drain(..));
self.maxs.extend(src.maxs.drain(..));
self.avgs.extend(src.avgs.drain(..));
@@ -414,7 +449,7 @@ where
.ts1s
.iter()
.zip(&item.ts2s)
.zip(&item.counts)
.zip(&item.cnts)
.zip(&item.mins)
.zip(&item.maxs)
.zip(&item.avgs)
@@ -514,7 +549,9 @@ where
panic!("TODO non-time-weighted binning to be impl");
}
} else {
error!("partially filled bin with cnt 0");
if self.filled_up_to != self.ts1now {
error!("partially filled bin with cnt 0");
}
}
}
if self.cnt == 0 && !push_empty {
@@ -522,7 +559,7 @@ where
} else {
self.out.ts1s.push_back(self.ts1now.ns());
self.out.ts2s.push_back(self.ts2now.ns());
self.out.counts.push_back(self.cnt);
self.out.cnts.push_back(self.cnt);
self.out.mins.push_back(self.min.clone());
self.out.maxs.push_back(self.max.clone());
self.out.avgs.push_back(self.avg as f32);
@@ -611,8 +648,9 @@ where
.as_any_ref()
.downcast_ref::<BinsDim0CollectedResult<netpod::EnumVariant>>()
{
let mins = self.mins.iter().map(|x| 0).collect();
let maxs = self.mins.iter().map(|x| 0).collect();
debug!("boxed_collected_with_enum_fix");
let mins = self.mins.iter().map(|x| 6).collect();
let maxs = self.mins.iter().map(|x| 7).collect();
let bins = BinsDim0CollectedResult::<u16> {
ts_anchor_sec: self.ts_anchor_sec.clone(),
ts1_off_ms: self.ts1_off_ms.clone(),
@@ -781,7 +819,7 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
let vals = self.vals.as_mut().unwrap();
vals.ts1s.append(&mut src.ts1s);
vals.ts2s.append(&mut src.ts2s);
vals.counts.append(&mut src.counts);
vals.cnts.append(&mut src.cnts);
vals.mins.append(&mut src.mins);
vals.maxs.append(&mut src.maxs);
vals.avgs.append(&mut src.avgs);
@@ -809,7 +847,7 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
let bin_count_exp = if let Some(r) = &binrange {
r.bin_count() as u32
} else {
warn!("no binrange given");
debug!("no binrange given");
0
};
let mut vals = if let Some(x) = self.vals.take() {
@@ -818,8 +856,8 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
return Err(Error::with_msg_no_trace("BinsDim0Collector without vals"));
};
let bin_count = vals.ts1s.len() as u32;
eprintln!(
"-------------- MAKE MISSING BINS bin_count_exp {} bin_count {}",
debug!(
"result make missing bins bin_count_exp {} bin_count {}",
bin_count_exp, bin_count
);
let (missing_bins, continue_at, finished_at) = if bin_count < bin_count_exp {
@@ -849,7 +887,7 @@ impl<NTY: ScalarOps> CollectorType for BinsDim0Collector<NTY> {
let ts2s = vals.ts2s.make_contiguous();
let (ts_anch, ts1ms, ts1ns) = ts_offs_from_abs(ts1s);
let (ts2ms, ts2ns) = ts_offs_from_abs_with_anchor(ts_anch, ts2s);
let counts = vals.counts;
let counts = vals.cnts;
let mins = vals.mins;
let maxs = vals.maxs;
let avgs = vals.avgs;
@@ -885,8 +923,8 @@ impl<NTY: ScalarOps> CollectableType for BinsDim0<NTY> {
#[derive(Debug)]
pub struct BinsDim0Aggregator<NTY> {
range: SeriesRange,
count: u64,
minmax: Option<(NTY, NTY)>,
cnt: u64,
minmaxlst: Option<(NTY, NTY, NTY)>,
sumc: u64,
sum: f32,
}
@@ -895,8 +933,8 @@ impl<NTY: ScalarOps> BinsDim0Aggregator<NTY> {
pub fn new(range: SeriesRange, _do_time_weight: bool) -> Self {
Self {
range,
count: 0,
minmax: None,
cnt: 0,
minmaxlst: None,
sumc: 0,
sum: 0f32,
}
@@ -914,30 +952,32 @@ impl<NTY: ScalarOps> TimeBinnableTypeAggregator for BinsDim0Aggregator<NTY> {
fn ingest(&mut self, item: &Self::Input) {
let beg = self.range.beg_u64();
let end = self.range.end_u64();
for (((((&ts1, &ts2), &count), min), max), &avg) in item
for ((((((&ts1, &ts2), &count), min), max), &avg), lst) in item
.ts1s
.iter()
.zip(item.ts2s.iter())
.zip(item.counts.iter())
.zip(item.cnts.iter())
.zip(item.mins.iter())
.zip(item.maxs.iter())
.zip(item.avgs.iter())
.zip(item.lsts.iter())
{
if count == 0 {
} else if ts2 <= beg {
if ts2 <= beg {
} else if ts1 >= end {
} else {
if let Some((cmin, cmax)) = self.minmax.as_mut() {
if let Some((cmin, cmax, clst)) = self.minmaxlst.as_mut() {
if min < cmin {
*cmin = min.clone();
}
if max > cmax {
*cmax = max.clone();
}
*clst = lst.clone();
} else {
self.minmax = Some((min.clone(), max.clone()));
self.minmaxlst = Some((min.clone(), max.clone(), lst.clone()));
}
self.count += count;
self.cnt += count;
// TODO this works only for equidistant bins edges.
self.sumc += 1;
self.sum += avg;
}
@@ -945,29 +985,33 @@ impl<NTY: ScalarOps> TimeBinnableTypeAggregator for BinsDim0Aggregator<NTY> {
}
fn result_reset(&mut self, range: SeriesRange) -> Self::Output {
let (min, max) = if let Some((min, max)) = self.minmax.take() {
(min, max)
let ret = if let Some((min, max, lst)) = self.minmaxlst.take() {
self.minmaxlst = Some((lst.clone(), lst.clone(), lst.clone()));
let avg = if self.sumc > 0 {
self.sum / self.sumc as f32
} else {
NTY::zero_b().as_prim_f32_b()
};
Self::Output {
ts1s: [self.range.beg_u64()].into(),
ts2s: [self.range.end_u64()].into(),
cnts: [self.cnt].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
lsts: [lst].into(),
// TODO
dim0kind: None,
}
} else {
(NTY::zero_b(), NTY::zero_b())
};
let avg = if self.sumc > 0 {
self.sum / self.sumc as f32
} else {
NTY::zero_b().as_prim_f32_b()
};
let ret = Self::Output {
ts1s: [self.range.beg_u64()].into(),
ts2s: [self.range.end_u64()].into(),
counts: [self.count].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
// TODO
dim0kind: None,
if self.cnt != 0 {
error!("result_reset non-zero cnt but no minmaxlst");
}
warn!("result_reset missing minmaxlst");
Self::Output::empty()
};
self.range = range;
self.count = 0;
self.minmax = None;
self.cnt = 0;
self.sumc = 0;
self.sum = 0.;
ret
@@ -1264,7 +1308,7 @@ impl<NTY: ScalarOps> TimeBinned for BinsDim0<NTY> {
fn counts(&self) -> &[u64] {
// TODO check for contiguous
self.counts.as_slices().0
self.cnts.as_slices().0
}
// TODO is Vec needed?
@@ -1288,7 +1332,7 @@ impl<NTY: ScalarOps> TimeBinned for BinsDim0<NTY> {
if self.ts1s.len() != self.ts2s.len() {
write!(&mut msg, "ts1s ≠ ts2s\n").unwrap();
}
for (i, ((count, min), max)) in self.counts.iter().zip(&self.mins).zip(&self.maxs).enumerate() {
for (i, ((count, min), max)) in self.cnts.iter().zip(&self.mins).zip(&self.maxs).enumerate() {
if min.as_prim_f32_b() < 1. && *count != 0 {
write!(&mut msg, "i {} count {} min {:?} max {:?}\n", i, count, min, max).unwrap();
}
@@ -1313,10 +1357,11 @@ impl<NTY: ScalarOps> TimeBinned for BinsDim0<NTY> {
let ret = BinsDim0::<f32> {
ts1s: replace(&mut self.ts1s, VecDeque::new()),
ts2s: replace(&mut self.ts2s, VecDeque::new()),
counts: replace(&mut self.counts, VecDeque::new()),
cnts: replace(&mut self.cnts, VecDeque::new()),
mins: self.mins.iter().map(AsPrimF32::as_prim_f32_b).collect(),
maxs: self.maxs.iter().map(AsPrimF32::as_prim_f32_b).collect(),
avgs: replace(&mut self.avgs, VecDeque::new()),
lsts: self.lsts.iter().map(AsPrimF32::as_prim_f32_b).collect(),
dim0kind: None,
};
Box::new(ret)
@@ -1328,7 +1373,7 @@ impl<NTY: ScalarOps> TimeBinned for BinsDim0<NTY> {
// TODO make it harder to forget new members when the struct may get modified in the future
dst.ts1s.extend(self.ts1s.drain(range.clone()));
dst.ts2s.extend(self.ts2s.drain(range.clone()));
dst.counts.extend(self.counts.drain(range.clone()));
dst.cnts.extend(self.cnts.drain(range.clone()));
dst.mins.extend(self.mins.drain(range.clone()));
dst.maxs.extend(self.maxs.drain(range.clone()));
dst.avgs.extend(self.avgs.drain(range.clone()));
@@ -1359,9 +1404,10 @@ fn bins_timebin_fill_empty_00() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
for i in 0..5 {
exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0.);
}
// Currently bins without lst can not exist.
// for i in 0..5 {
// exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0., None);
// }
assert_eq!(got, &exp);
}
@@ -1384,9 +1430,10 @@ fn bins_timebin_fill_empty_01() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
for i in 0..5 {
exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0.);
}
// Currently bins without lst can not exist.
// for i in 0..5 {
// exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0., None);
// }
assert_eq!(got, &exp);
}
@@ -1408,9 +1455,10 @@ fn bins_timebin_push_empty_00() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
for i in 0..1 {
exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0.);
}
// Currently bins without lst can not exist.
// for i in 0..1 {
// exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0., None);
// }
assert_eq!(got, &exp);
}
@@ -1434,17 +1482,18 @@ fn bins_timebin_push_empty_01() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
for i in 0..3 {
exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0.);
}
// Currently bins without lst can not exist.
// for i in 0..3 {
// exp.push(SEC * 2 * (9 + i), SEC * 2 * (10 + i), 0, 0, 0, 0., None);
// }
assert_eq!(got, &exp);
}
#[test]
fn bins_timebin_ingest_only_before() {
let mut bins = BinsDim0::<u32>::empty();
bins.push(SEC * 2, SEC * 4, 3, 7, 9, 8.1);
bins.push(SEC * 4, SEC * 6, 3, 6, 9, 8.2);
bins.push(SEC * 2, SEC * 4, 3, 7, 9, 8.1, 8);
bins.push(SEC * 4, SEC * 6, 3, 6, 9, 8.2, 8);
let binrange = BinnedRangeEnum::Time(BinnedRange {
bin_len: TsNano::from_ns(SEC * 2),
bin_off: 9,
@@ -1460,16 +1509,17 @@ fn bins_timebin_ingest_only_before() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0.);
// Currently bins without lst can not exist.
// exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None);
assert_eq!(got, &exp);
}
#[test]
fn bins_timebin_ingest_00() {
let mut bins = BinsDim0::<u32>::empty();
bins.push(SEC * 20, SEC * 21, 3, 70, 94, 82.);
bins.push(SEC * 21, SEC * 22, 5, 71, 93, 86.);
bins.push(SEC * 23, SEC * 24, 6, 72, 92, 81.);
bins.push(SEC * 20, SEC * 21, 3, 70, 94, 82., 80);
bins.push(SEC * 21, SEC * 22, 5, 71, 93, 86., 81);
bins.push(SEC * 23, SEC * 24, 6, 72, 92, 81., 82);
let binrange = BinnedRangeEnum::Time(BinnedRange {
bin_len: TsNano::from_ns(SEC * 2),
bin_off: 9,
@@ -1485,9 +1535,10 @@ fn bins_timebin_ingest_00() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0.);
exp.push(SEC * 20, SEC * 22, 8, 70, 94, 84.);
exp.push(SEC * 22, SEC * 24, 6, 72, 92, 81.);
// Currently bins without lst can not exist.
// exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None);
exp.push(SEC * 20, SEC * 22, 8, 70, 94, 84., 82);
exp.push(SEC * 22, SEC * 24, 6, 72, 92, 81., 91);
assert_eq!(got, &exp);
}
@@ -1500,7 +1551,7 @@ fn bins_timebin_ingest_continuous_00() {
});
let do_time_weight = true;
let mut bins = BinsDim0::<u32>::empty();
bins.push(SEC * 20, SEC * 21, 3, 70, 94, 82.);
bins.push(SEC * 20, SEC * 21, 3, 70, 94, 82., 80);
//bins.push(SEC * 21, SEC * 22, 5, 71, 93, 86.);
//bins.push(SEC * 23, SEC * 24, 6, 72, 92, 81.);
let mut binner = bins
@@ -1512,8 +1563,9 @@ fn bins_timebin_ingest_continuous_00() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0.);
exp.push(SEC * 20, SEC * 22, 8, 70, 94, 84.);
exp.push(SEC * 22, SEC * 24, 6, 72, 92, 81.);
// Currently bins without lst can not exist.
// exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None);
exp.push(SEC * 20, SEC * 22, 8, 70, 94, 84., 82);
exp.push(SEC * 22, SEC * 24, 6, 72, 92, 81., 91);
assert_eq!(got, &exp);
}

4
crates/items_2/src/binsxbindim0.rs
View File

@@ -886,10 +886,11 @@ impl<NTY: ScalarOps> TimeBinned for BinsXbinDim0<NTY> {
let ret = super::binsdim0::BinsDim0::<f32> {
ts1s: replace(&mut self.ts1s, VecDeque::new()),
ts2s: replace(&mut self.ts2s, VecDeque::new()),
counts: replace(&mut self.counts, VecDeque::new()),
cnts: replace(&mut self.counts, VecDeque::new()),
mins: self.mins.iter().map(AsPrimF32::as_prim_f32_b).collect(),
maxs: self.maxs.iter().map(AsPrimF32::as_prim_f32_b).collect(),
avgs: replace(&mut self.avgs, VecDeque::new()),
lsts: err::todoval(),
dim0kind: None,
};
Box::new(ret)
@@ -905,6 +906,7 @@ impl<NTY: ScalarOps> TimeBinned for BinsXbinDim0<NTY> {
dst.mins.extend(self.mins.drain(range.clone()));
dst.maxs.extend(self.maxs.drain(range.clone()));
dst.avgs.extend(self.avgs.drain(range.clone()));
todo!("handle last_seen");
Ok(())
} else {
let type_name = any::type_name::<Self>();

188
crates/items_2/src/eventsdim0.rs
View File

@@ -57,31 +57,22 @@ use std::fmt;
use std::mem;
#[allow(unused)]
macro_rules! trace_ingest {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
macro_rules! trace_init { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[allow(unused)]
macro_rules! trace_ingest_item {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
macro_rules! trace_ingest { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[allow(unused)]
macro_rules! trace2 {
($($arg:tt)*) => {};
($($arg:tt)*) => { trace!($($arg)*); };
}
macro_rules! trace_ingest_item { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[allow(unused)]
macro_rules! trace_binning {
($($arg:tt)*) => {
if false {
trace!($($arg)*);
}
};
}
macro_rules! trace2 { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[allow(unused)]
macro_rules! trace_binning { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[allow(unused)]
macro_rules! debug_ingest { ($($arg:tt)*) => ( if true { trace!($($arg)*); }) }
#[derive(Clone, PartialEq, Serialize, Deserialize)]
pub struct EventsDim0NoPulse<STY> {
@@ -545,12 +536,11 @@ impl<STY: ScalarOps> items_0::collect_s::CollectableType for EventsDim0<STY> {
pub struct EventsDim0Aggregator<STY> {
range: SeriesRange,
count: u64,
minmax: Option<(STY, STY)>,
minmaxlst: Option<(STY, STY, STY)>,
sumc: u64,
sum: f32,
int_ts: u64,
last_ts: u64,
last_val: Option<STY>,
do_time_weight: bool,
events_ignored_count: u64,
items_seen: usize,
@@ -580,14 +570,13 @@ impl<STY: ScalarOps> TimeAggregatorCommonV0Trait for EventsDim0Aggregator<STY> {
self.apply_event_unweight(val.clone());
self.count += 1;
self.last_ts = ts;
self.last_val = Some(val.clone());
}
}
fn common_ingest_one_before(&mut self, item: &Self::Input, j: usize) {
//trace_ingest!("{self_name} ingest {:6} {:20} {:10?} BEFORE", i1, ts, val);
self.apply_min_max_lst(item.values[j].clone());
self.last_ts = item.tss[j];
self.last_val = Some(item.values[j].clone());
}
fn common_ingest_range(&mut self, item: &Self::Input, r: core::ops::Range<usize>) {
@@ -598,7 +587,6 @@ impl<STY: ScalarOps> TimeAggregatorCommonV0Trait for EventsDim0Aggregator<STY> {
}
self.count += 1;
self.last_ts = ts;
self.last_val = Some(val.clone());
}
}
}
@@ -609,16 +597,16 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
}
pub fn new(range: SeriesRange, do_time_weight: bool) -> Self {
trace_init!("{}::new", Self::type_name());
let int_ts = range.beg_u64();
Self {
range,
count: 0,
minmax: None,
minmaxlst: None,
sumc: 0,
sum: 0.,
int_ts,
last_ts: 0,
last_val: None,
do_time_weight,
events_ignored_count: 0,
items_seen: 0,
@@ -626,24 +614,24 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
}
// TODO reduce clone.. optimize via more traits to factor the trade-offs?
fn apply_min_max(&mut self, val: STY) {
fn apply_min_max_lst(&mut self, val: STY) {
trace_ingest!(
"apply_min_max val {:?} last_val {:?} count {} sumc {:?} minmax {:?}",
"apply_min_max_lst val {:?} count {} sumc {:?} minmaxlst {:?}",
val,
self.last_val,
self.count,
self.sumc,
self.minmax,
self.minmaxlst,
);
if let Some((min, max)) = self.minmax.as_mut() {
if let Some((min, max, lst)) = self.minmaxlst.as_mut() {
if *min > val {
*min = val.clone();
}
if *max < val {
*max = val.clone();
}
*lst = val.clone();
} else {
self.minmax = Some((val.clone(), val.clone()));
self.minmaxlst = Some((val.clone(), val.clone(), val.clone()));
}
}
@@ -651,7 +639,7 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
error!("TODO check again result_reset_unweight");
err::todo();
let vf = val.as_prim_f32_b();
self.apply_min_max(val);
self.apply_min_max_lst(val);
if vf.is_nan() {
} else {
self.sum += vf;
@@ -660,11 +648,11 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
}
fn apply_event_time_weight(&mut self, px: u64) {
if let Some(v) = &self.last_val {
if let Some((_, _, v)) = self.minmaxlst.as_ref() {
trace_ingest!("apply_event_time_weight with v {v:?}");
let vf = v.as_prim_f32_b();
let v2 = v.clone();
self.apply_min_max(v2);
self.apply_min_max_lst(v2);
self.sumc += 1;
let w = (px - self.int_ts) as f32 * 1e-9;
if false {
@@ -683,7 +671,7 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
}
self.int_ts = px;
} else {
debug!("apply_event_time_weight NO VALUE");
debug_ingest!("apply_event_time_weight NO VALUE");
}
}
@@ -695,44 +683,47 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
TimeAggregatorCommonV0Func::ingest_time_weight(self, item)
}
fn reset_values(&mut self, range: SeriesRange) {
fn reset_values(&mut self, lst: STY, range: SeriesRange) {
self.int_ts = range.beg_u64();
trace_binning!("ON RESET SET int_ts {:10}", self.int_ts);
trace_init!("ON RESET SET int_ts {:10}", self.int_ts);
self.range = range;
self.count = 0;
self.sum = 0.;
self.sumc = 0;
self.minmax = None;
self.minmaxlst = Some((lst.clone(), lst.clone(), lst));
self.items_seen = 0;
}
fn result_reset_unweight(&mut self, range: SeriesRange) -> BinsDim0<STY> {
let (min, max) = if let Some((min, max)) = self.minmax.take() {
(min, max)
error!("TODO result_reset_unweight");
panic!("TODO result_reset_unweight");
if let Some((min, max, lst)) = self.minmaxlst.take() {
let avg = if self.sumc > 0 {
self.sum / self.sumc as f32
} else {
STY::zero_b().as_prim_f32_b()
};
let ret = if self.range.is_time() {
BinsDim0 {
ts1s: [self.range.beg_u64()].into(),
ts2s: [self.range.end_u64()].into(),
cnts: [self.count].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
lsts: [lst.clone()].into(),
dim0kind: Some(self.range.dim0kind()),
}
} else {
error!("TODO result_reset_unweight");
err::todoval()
};
self.reset_values(lst, range);
ret
} else {
(STY::zero_b(), STY::zero_b())
};
let avg = if self.sumc > 0 {
self.sum / self.sumc as f32
} else {
STY::zero_b().as_prim_f32_b()
};
let ret = if self.range.is_time() {
BinsDim0 {
ts1s: [self.range.beg_u64()].into(),
ts2s: [self.range.end_u64()].into(),
counts: [self.count].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
dim0kind: Some(self.range.dim0kind()),
}
} else {
error!("TODO result_reset_unweight");
err::todoval()
};
self.reset_values(range);
ret
// TODO add check that nothing is different from initial values, or reset without lst.
BinsDim0::empty()
}
}
fn result_reset_time_weight(&mut self, range: SeriesRange) -> BinsDim0<STY> {
@@ -751,36 +742,33 @@ impl<STY: ScalarOps> EventsDim0Aggregator<STY> {
error!("TODO result_reset_time_weight");
err::todoval()
}
let (min, max) = if let Some((min, max)) = self.minmax.take() {
(min, max)
} else {
(STY::zero_b(), STY::zero_b())
};
let avg = if self.sumc > 0 {
self.sum / (self.range.delta_u64() as f32 * 1e-9)
} else {
if let Some(v) = self.last_val.as_ref() {
v.as_prim_f32_b()
if let Some((min, max, lst)) = self.minmaxlst.take() {
let avg = if self.sumc > 0 {
self.sum / (self.range.delta_u64() as f32 * 1e-9)
} else {
STY::zero_b().as_prim_f32_b()
}
};
let ret = if self.range.is_time() {
BinsDim0 {
ts1s: [range_beg].into(),
ts2s: [range_end].into(),
counts: [self.count].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
dim0kind: Some(self.range.dim0kind()),
}
lst.as_prim_f32_b()
};
let ret = if self.range.is_time() {
BinsDim0 {
ts1s: [range_beg].into(),
ts2s: [range_end].into(),
cnts: [self.count].into(),
mins: [min].into(),
maxs: [max].into(),
avgs: [avg].into(),
lsts: [lst.clone()].into(),
dim0kind: Some(self.range.dim0kind()),
}
} else {
error!("TODO result_reset_time_weight");
err::todoval()
};
self.reset_values(lst, range);
ret
} else {
error!("TODO result_reset_time_weight");
err::todoval()
};
self.reset_values(range);
ret
// TODO add check that nothing is different from initial values, or reset without lst.
BinsDim0::empty()
}
}
}
@@ -793,9 +781,7 @@ impl<STY: ScalarOps> TimeBinnableTypeAggregator for EventsDim0Aggregator<STY> {
}
fn ingest(&mut self, item: &Self::Input) {
if true {
trace_ingest!("{} ingest {} events", Self::type_name(), item.len());
}
trace_ingest!("{} ingest {} events", Self::type_name(), item.len());
if false {
for (i, &ts) in item.tss.iter().enumerate() {
trace_ingest!("{} ingest {:6} {:20}", Self::type_name(), i, ts);
@@ -825,6 +811,10 @@ impl<STY: ScalarOps> TimeBinnable for EventsDim0<STY> {
do_time_weight: bool,
emit_empty_bins: bool,
) -> Box<dyn TimeBinner> {
trace_init!(
"<{} as items_0::timebin::TimeBinnable>::time_binner_new",
self.type_name()
);
// TODO get rid of unwrap
let ret = EventsDim0TimeBinner::<STY>::new(binrange, do_time_weight, emit_empty_bins).unwrap();
Box::new(ret)
@@ -1095,11 +1085,11 @@ impl<STY: ScalarOps> EventsDim0TimeBinner<STY> {
}
pub fn new(binrange: BinnedRangeEnum, do_time_weight: bool, emit_empty_bins: bool) -> Result<Self, Error> {
trace!("{}::new binrange {:?}", Self::type_name(), binrange);
trace_init!("{}::new binrange {:?}", Self::type_name(), binrange);
let rng = binrange
.range_at(0)
.ok_or_else(|| Error::with_msg_no_trace("empty binrange"))?;
trace!("{}::new rng {:?}", Self::type_name(), rng);
trace_init!("{}::new rng {:?}", Self::type_name(), rng);
let agg = EventsDim0Aggregator::new(rng, do_time_weight);
let ret = Self {
binrange,
@@ -1480,7 +1470,7 @@ fn events_timebin_ingest_continuous_00() {
let got = ready.unwrap();
let got: &BinsDim0<u32> = got.as_any_ref().downcast_ref().unwrap();
let mut exp = BinsDim0::empty();
exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0.);
exp.push(SEC * 20, SEC * 22, 1, 20, 20, 20.);
// exp.push(SEC * 18, SEC * 20, 0, 0, 0, 0., None);
exp.push(SEC * 20, SEC * 22, 1, 20, 20, 20., 20);
assert!(f32_iter_cmp_near(got.avgs.clone(), exp.avgs.clone(), 0.0001, 0.0001));
}

25
crates/netpod/src/netpod.rs
View File

@@ -702,6 +702,12 @@ impl Default for EnumVariant {
}
}
impl fmt::Display for EnumVariant {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{}({})", self.ix, self.name)
}
}
impl AppendToUrl for ScalarType {
fn append_to_url(&self, url: &mut Url) {
let mut g = url.query_pairs_mut();
@@ -2141,6 +2147,16 @@ const TIME_BIN_THRESHOLDS: [u64; 26] = [
DAY * 64,
];
const TIME_BIN_LEN_CACHE_OPTS: [DtMs; 2] = [
//
DtMs(1000 * 10),
DtMs(1000 * 60 * 60),
];
pub fn time_bin_len_cache_opts() -> &'static [DtMs] {
&TIME_BIN_LEN_CACHE_OPTS
}
const PULSE_BIN_THRESHOLDS: [u64; 25] = [
10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000, 200000, 400000,
800000, 1000000, 2000000, 4000000, 8000000, 10000000,
@@ -2485,7 +2501,7 @@ where
}*/
let beg = self.bin_len.times(self.bin_off).as_u64();
let end = self.bin_len.times(self.bin_off + self.bin_cnt).as_u64();
warn!("TODO make generic for pulse");
debug!("TODO make generic for pulse");
NanoRange { beg, end }
}
@@ -3715,6 +3731,13 @@ impl ChannelTypeConfigGen {
ChannelTypeConfigGen::SfDatabuffer(x) => x.shape(),
}
}
pub fn series(&self) -> Option<u64> {
match self {
ChannelTypeConfigGen::Scylla(ch_conf) => Some(ch_conf.series()),
ChannelTypeConfigGen::SfDatabuffer(sf_ch_fetch_info) => None,
}
}
}
impl From<SfChFetchInfo> for ChannelTypeConfigGen {

8
crates/netpod/src/query.rs
View File

@@ -30,6 +30,7 @@ pub enum CacheUsage {
Use,
Ignore,
Recreate,
V0NoCache,
}
impl CacheUsage {
@@ -38,6 +39,7 @@ impl CacheUsage {
CacheUsage::Use => "use",
CacheUsage::Ignore => "ignore",
CacheUsage::Recreate => "recreate",
CacheUsage::V0NoCache => "v0nocache",
}
.into()
}
@@ -53,6 +55,8 @@ impl CacheUsage {
Ok(Some(CacheUsage::Ignore))
} else if k == "recreate" {
Ok(Some(CacheUsage::Recreate))
} else if k == "v0nocache" {
Ok(Some(CacheUsage::V0NoCache))
} else {
Err(Error::with_msg(format!("unexpected cacheUsage {:?}", k)))?
}
@@ -67,6 +71,8 @@ impl CacheUsage {
CacheUsage::Recreate
} else if s == "use" {
CacheUsage::Use
} else if s == "v0nocache" {
CacheUsage::V0NoCache
} else {
return Err(Error::with_msg(format!("can not interpret cache usage string: {}", s)));
};
@@ -78,6 +84,7 @@ impl CacheUsage {
CacheUsage::Use => true,
CacheUsage::Ignore => false,
CacheUsage::Recreate => true,
CacheUsage::V0NoCache => false,
}
}
@@ -86,6 +93,7 @@ impl CacheUsage {
CacheUsage::Use => true,
CacheUsage::Ignore => false,
CacheUsage::Recreate => false,
CacheUsage::V0NoCache => false,
}
}
}

14
crates/nodenet/src/scylla.rs
View File

@@ -163,13 +163,13 @@ impl Stream for ScyllaEventsReadStream {
} else if let Some(fut) = self.stream.as_mut() {
match fut.poll_next_unpin(cx) {
Ready(Some(x)) => {
let x = try_map_sitemty_data!(x, |x| match x {
ChannelEvents::Events(x) => {
let x = x.to_dim0_f32_for_binning();
Ok(ChannelEvents::Events(x))
}
ChannelEvents::Status(x) => Ok(ChannelEvents::Status(x)),
});
// let x = try_map_sitemty_data!(x, |x| match x {
// ChannelEvents::Events(x) => {
// let x = x.to_dim0_f32_for_binning();
// Ok(ChannelEvents::Events(x))
// }
// ChannelEvents::Status(x) => Ok(ChannelEvents::Status(x)),
// });
Ready(Some(x))
}
Ready(None) => Ready(None),

4
crates/query/src/api4/binned.rs
View File

@@ -134,8 +134,8 @@ impl BinnedQuery {
&self.transform
}
pub fn cache_usage(&self) -> CacheUsage {
self.cache_usage.as_ref().map_or(CacheUsage::Ignore, |x| x.clone())
pub fn cache_usage(&self) -> Option<CacheUsage> {
self.cache_usage.clone()
}
pub fn disk_stats_every(&self) -> ByteSize {

11
crates/scyllaconn/src/bincache.rs
View File

@@ -245,14 +245,15 @@ pub async fn worker_write(
scy: &ScySession,
) -> Result<(), streams::timebin::cached::reader::Error> {
let mut msp_last = u64::MAX;
for (((((&ts1, &ts2), &cnt), &min), &max), &avg) in bins
for ((((((&ts1, &ts2), &cnt), &min), &max), &avg), &lst) in bins
.ts1s
.iter()
.zip(bins.ts2s.iter())
.zip(bins.counts.iter())
.zip(bins.cnts.iter())
.zip(bins.mins.iter())
.zip(bins.maxs.iter())
.zip(bins.avgs.iter())
.zip(bins.lsts.iter())
{
let bin_len = DtMs::from_ms_u64((ts2 - ts1) / 1000000);
let div = streams::timebin::cached::reader::part_len(bin_len).ns();
@@ -267,6 +268,7 @@ pub async fn worker_write(
min,
max,
avg,
lst,
);
// trace!("cache write {:?}", params);
scy.execute(stmts_cache.st_write_f32(), params)
@@ -296,7 +298,7 @@ pub async fn worker_read(
.execute_iter(stmts_cache.st_read_f32().clone(), params)
.await
.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
let mut it = res.into_typed::<(i32, i64, f32, f32, f32)>();
let mut it = res.into_typed::<(i32, i64, f32, f32, f32, f32)>();
let mut bins = BinsDim0::empty();
while let Some(x) = it.next().await {
let row = x.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
@@ -305,9 +307,10 @@ pub async fn worker_read(
let min = row.2;
let max = row.3;
let avg = row.4;
let lst = row.5;
let ts1 = bin_len.ns() * off + div * msp;
let ts2 = ts1 + bin_len.ns();
bins.push(ts1, ts2, cnt, min, max, avg);
bins.push(ts1, ts2, cnt, min, max, avg, lst);
}
Ok(bins)
}

2
crates/scyllaconn/src/conn.rs
View File

@@ -17,7 +17,7 @@ pub async fn create_scy_session(scyconf: &ScyllaConfig) -> Result<Arc<ScySession
}
pub async fn create_scy_session_no_ks(scyconf: &ScyllaConfig) -> Result<ScySession, Error> {
warn!("create_connection\n\n CREATING SCYLLA CONNECTION\n\n");
warn!("creating scylla connection");
let scy = scylla::SessionBuilder::new()
.known_nodes(&scyconf.hosts)
.default_execution_profile_handle(

6
crates/scyllaconn/src/events2/prepare.rs
View File

@@ -262,8 +262,8 @@ impl StmtsCache {
.prepare(format!(
concat!(
"insert into {}.{}binned_scalar_f32",
" (series, bin_len_ms, ts_msp, off, count, min, max, avg)",
" values (?, ?, ?, ?, ?, ?, ?, ?)"
" (series, bin_len_ms, ts_msp, off, count, min, max, avg, lst)",
" values (?, ?, ?, ?, ?, ?, ?, ?, ?)"
),
ks,
rt.table_prefix()
@@ -272,7 +272,7 @@ impl StmtsCache {
let st_read_f32 = scy
.prepare(format!(
concat!(
"select off, count, min, max, avg",
"select off, count, min, max, avg, lst",
" from {}.{}binned_scalar_f32",
" where series = ?",
" and bin_len_ms = ?",

15
crates/scyllaconn/src/schema.rs Normal file
View File

@@ -0,0 +1,15 @@
use err::thiserror;
use err::ThisError;
use netpod::ttl::RetentionTime;
use netpod::ScyllaConfig;
use scylla::Session as ScySession;
#[derive(Debug, ThisError)]
#[cstm(name = "ScyllaSchema")]
pub enum Error {
Scylla,
}
pub async fn schema(rt: RetentionTime, scyco: &ScyllaConfig, scy: &ScySession) -> Result<(), Error> {
todo!()
}

1
crates/scyllaconn/src/scyllaconn.rs
View File

@@ -5,6 +5,7 @@ pub mod errconv;
pub mod events;
pub mod events2;
pub mod range;
pub mod schema;
pub mod status;
pub mod worker;

4
crates/scyllaconn/src/worker.rs
View File

@@ -33,6 +33,7 @@ pub enum Error {
Toplist(#[from] crate::accounting::toplist::Error),
MissingKeyspaceConfig,
CacheWriteF32(#[from] streams::timebin::cached::reader::Error),
Schema(#[from] crate::schema::Error),
}
#[derive(Debug)]
@@ -212,6 +213,9 @@ impl ScyllaWorker {
.await
.map_err(Error::ScyllaConnection)?;
let scy = Arc::new(scy);
crate::schema::schema(RetentionTime::Short, &self.scyconf_st, &scy).await?;
crate::schema::schema(RetentionTime::Medium, &self.scyconf_mt, &scy).await?;
crate::schema::schema(RetentionTime::Long, &self.scyconf_lt, &scy).await?;
let kss = [
self.scyconf_st.keyspace.as_str(),
self.scyconf_mt.keyspace.as_str(),

15
crates/streams/src/test/timebin.rs
View File

@@ -64,14 +64,15 @@ fn time_bin_00() -> Result<(), Error> {
let bins = BinsDim0::empty();
d.push_back(bins);
let mut bins = BinsDim0::empty();
bins.push(SEC * 0, SEC * 1, 0, 0.0, 0.0, 0.0);
bins.push(SEC * 1, SEC * 2, 2, 0.0535830, 100.0589, 50.05624);
bins.push(SEC * 2, SEC * 3, 2, 200.06143, 300.07645, 250.06894);
bins.push(SEC * 3, SEC * 4, 2, 400.08554, 500.05222, 450.06888);
bins.push(SEC * 4, SEC * 5, 2, 600.0025, 700.09094, 650.04675);
// Currently can not cosntruct bins without minmaxlst
// bins.push(SEC * 0, SEC * 1, 0, 0.0, 0.0, 0.0);
bins.push(SEC * 1, SEC * 2, 2, 0.0535830, 100.0589, 50.05624, 100.0589);
bins.push(SEC * 2, SEC * 3, 2, 200.06143, 300.07645, 250.06894, 300.07645);
bins.push(SEC * 3, SEC * 4, 2, 400.08554, 500.05222, 450.06888, 500.05222);
bins.push(SEC * 4, SEC * 5, 2, 600.0025, 700.09094, 650.04675, 700.09094);
d.push_back(bins);
let mut bins = BinsDim0::empty();
bins.push(SEC * 5, SEC * 6, 2, 800.0619, 900.02844, 850.04517);
bins.push(SEC * 5, SEC * 6, 2, 800.0619, 900.02844, 850.04517, 900.02844);
d.push_back(bins);
d
};
@@ -342,6 +343,7 @@ fn timebin_multi_stage_00() -> Result<(), Error> {
20 + 2 * i as i32,
21 + 2 * i as i32,
20.5 + 2. * i as f32,
21 + 2 * i as i32,
);
}
bins
@@ -356,6 +358,7 @@ fn timebin_multi_stage_00() -> Result<(), Error> {
20 + 4 * i as i32,
23 + 4 * i as i32,
21.5 + 4. * i as f32,
23 + 4 * i as i32,
);
}
bins

11
crates/streams/src/timebin/fromevents.rs
View File

@@ -7,6 +7,7 @@ use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_2::binsdim0::BinsDim0;
use items_2::channelevents::ChannelEvents;
use netpod::log::*;
use netpod::BinnedRange;
use netpod::ChConf;
@@ -40,6 +41,16 @@ impl BinnedFromEvents {
panic!();
}
let stream = read_provider.read(evq, chconf);
let stream = stream.map(|x| {
let x = items_0::try_map_sitemty_data!(x, |x| match x {
ChannelEvents::Events(x) => {
let x = x.to_dim0_f32_for_binning();
Ok(ChannelEvents::Events(x))
}
ChannelEvents::Status(x) => Ok(ChannelEvents::Status(x)),
});
x
});
let stream = Box::pin(stream);
let stream = super::basic::TimeBinnedStream::new(stream, netpod::BinnedRangeEnum::Time(range), do_time_weight);
let stream = stream.map(|item| match item {

15
crates/streams/src/timebin/fromlayers.rs
View File

@@ -40,6 +40,8 @@ pub enum Error {
GapFill(#[from] super::gapfill::Error),
BinnedFromEvents(#[from] super::fromevents::Error),
SfDatabufferNotSupported,
#[error("FinerGridMismatch({0}, {1})")]
FinerGridMismatch(DtMs, DtMs),
}
type BoxedInput = Pin<Box<dyn Stream<Item = Sitemty<BinsDim0<f32>>> + Send>>;
@@ -75,7 +77,7 @@ impl TimeBinnedFromLayers {
cache_read_provider: Arc<dyn CacheReadProvider>,
events_read_provider: Arc<dyn EventsReadProvider>,
) -> Result<Self, Error> {
info!(
debug!(
"{}::new {:?} {:?} {:?}",
Self::type_name(),
series,
@@ -84,7 +86,7 @@ impl TimeBinnedFromLayers {
);
let bin_len = DtMs::from_ms_u64(range.bin_len.ms());
if bin_len_layers.contains(&bin_len) {
info!("{}::new bin_len in layers {:?}", Self::type_name(), range);
debug!("{}::new bin_len in layers {:?}", Self::type_name(), range);
let inp = super::gapfill::GapFill::new(
"FromLayers".into(),
ch_conf.clone(),
@@ -114,8 +116,11 @@ impl TimeBinnedFromLayers {
} else {
match find_next_finer_bin_len(bin_len, &bin_len_layers) {
Some(finer) => {
if bin_len.ms() % finer.ms() != 0 {
return Err(Error::FinerGridMismatch(bin_len, finer));
}
let range_finer = BinnedRange::from_nano_range(range.to_nano_range(), finer);
warn!(
debug!(
"{}::new next finer from bins {:?} {:?}",
Self::type_name(),
finer,
@@ -154,7 +159,7 @@ impl TimeBinnedFromLayers {
Ok(ret)
}
None => {
warn!("{}::new next finer from events", Self::type_name());
debug!("{}::new next finer from events", Self::type_name());
let series_range = SeriesRange::TimeRange(range.to_nano_range());
let one_before_range = true;
let select = EventsSubQuerySelect::new(
@@ -183,7 +188,7 @@ impl TimeBinnedFromLayers {
open_bytes,
inp: Box::pin(inp),
};
warn!("{}::new setup from events", Self::type_name());
debug!("{}::new setup from events", Self::type_name());
Ok(ret)
}
ChannelTypeConfigGen::SfDatabuffer(_) => return Err(Error::SfDatabufferNotSupported),

22
crates/streams/src/timebin/gapfill.rs
View File

@@ -305,7 +305,7 @@ impl GapFill {
}
let aa = &self.bins_for_cache_write;
if aa.len() >= 2 {
for (i, (&c1, &_c2)) in aa.counts.iter().rev().zip(aa.counts.iter().rev().skip(1)).enumerate() {
for (i, (&c1, &_c2)) in aa.cnts.iter().rev().zip(aa.cnts.iter().rev().skip(1)).enumerate() {
if c1 != 0 {
let n = aa.len() - (1 + i);
debug_cache!("{} cache_write_on_end consider {} for write", self.dbgname, n);
@@ -322,7 +322,7 @@ impl GapFill {
fn cache_write_intermediate(mut self: Pin<&mut Self>) -> Result<(), Error> {
let aa = &self.bins_for_cache_write;
if aa.len() >= 2 {
for (i, (&c1, &_c2)) in aa.counts.iter().rev().zip(aa.counts.iter().rev().skip(1)).enumerate() {
for (i, (&c1, &_c2)) in aa.cnts.iter().rev().zip(aa.cnts.iter().rev().skip(1)).enumerate() {
if c1 != 0 {
let n = aa.len() - (1 + i);
debug_cache!("{} cache_write_intermediate consider {} for write", self.dbgname, n);
@@ -462,9 +462,9 @@ impl Stream for GapFill {
beg: j.ns(),
end: self.range.full_range().end(),
};
warn!(
"----- RECEIVED SOMETHING, BUT NOT ALL, setup rest from finer {} {} {}",
self.range, j, range
debug!(
"{} received something but not all, setup rest from finer {} {} {}",
self.dbgname, self.range, j, range
);
match self.as_mut().setup_inp_finer(range, false) {
Ok(()) => {
@@ -473,14 +473,14 @@ impl Stream for GapFill {
Err(e) => Ready(Some(Err(::err::Error::from_string(e)))),
}
} else {
info!("----- RECEIVED EVERYTHING");
debug!("{} received everything", self.dbgname);
Ready(None)
}
} else {
let range = self.range.to_nano_range();
warn!(
"----- RECEIVED NOTHING SO FAR AT ALL, setup full range from finer {} {}",
self.range, range
debug!(
"{} received nothing at all, setup full range from finer {} {}",
self.dbgname, self.range, range
);
match self.as_mut().setup_inp_finer(range, false) {
Ok(()) => {
@@ -495,10 +495,10 @@ impl Stream for GapFill {
} else {
self.done = true;
if self.inp_finer_range_final_cnt == self.inp_finer_range_final_max {
trace_handle!("{} RANGE FINAL ALL", self.dbgname);
trace_handle!("{} range finale all", self.dbgname);
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))))
} else {
trace_handle!("{} SUBSTREAMS NOT FINAL", self.dbgname);
trace_handle!("{} substreams not final", self.dbgname);
continue;
}
};

66
crates/streams/src/timebinnedjson.rs
View File

@@ -52,7 +52,6 @@ fn assert_stream_send<'u, R>(stream: impl 'u + Send + Stream<Item = R>) -> impl
stream
}
// TODO factor out, it is use now also from GapFill.
pub async fn timebinnable_stream(
range: NanoRange,
one_before_range: bool,
@@ -92,6 +91,7 @@ pub async fn timebinnable_stream(
on_sitemty_data!(k, |k| {
let k: Box<dyn Events> = Box::new(k);
// trace!("got len {}", k.len());
let k = k.to_dim0_f32_for_binning();
let k = tr.0.transform(k);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k)))
})
@@ -316,19 +316,24 @@ async fn timebinned_stream(
events_read_provider: Option<Arc<dyn EventsReadProvider>>,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>, Error> {
use netpod::query::CacheUsage;
match (query.cache_usage(), cache_read_provider, events_read_provider) {
(CacheUsage::Use | CacheUsage::Recreate, Some(cache_read_provider), Some(events_read_provider)) => {
let series = if let Some(x) = query.channel().series() {
x
} else {
return Err(Error::with_msg_no_trace(
"cached time binned only available given a series id",
));
};
info!("--- CACHING PATH ---");
info!("{query:?}");
info!("subgrids {:?}", query.subgrids());
let range = binned_range.binned_range_time().to_nano_range();
let cache_usage = query.cache_usage().unwrap_or(CacheUsage::Use);
match (
ch_conf.series(),
cache_usage.clone(),
cache_read_provider,
events_read_provider,
) {
(
Some(series),
CacheUsage::Use | CacheUsage::Recreate | CacheUsage::Ignore,
Some(cache_read_provider),
Some(events_read_provider),
) => {
debug!(
"timebinned_stream caching {:?} subgrids {:?}",
query,
query.subgrids()
);
let do_time_weight = true;
let bin_len_layers = if let Some(subgrids) = query.subgrids() {
subgrids
@@ -336,16 +341,11 @@ async fn timebinned_stream(
.map(|&x| DtMs::from_ms_u64(1000 * x.as_secs()))
.collect()
} else {
vec![
DtMs::from_ms_u64(1000 * 10),
DtMs::from_ms_u64(1000 * 60 * 60),
// DtMs::from_ms_u64(1000 * 60 * 60 * 12),
// DtMs::from_ms_u64(1000 * 10),
]
netpod::time_bin_len_cache_opts().to_vec()
};
let stream = crate::timebin::TimeBinnedFromLayers::new(
ch_conf,
query.cache_usage(),
cache_usage,
query.transform().clone(),
EventsSubQuerySettings::from(&query),
query.log_level().into(),
@@ -369,10 +369,8 @@ async fn timebinned_stream(
}
_ => {
let range = binned_range.binned_range_time().to_nano_range();
let do_time_weight = true;
let one_before_range = true;
let stream = timebinnable_stream(
range,
one_before_range,
@@ -419,7 +417,7 @@ pub async fn timebinned_json(
let deadline = Instant::now()
+ query
.timeout_content()
.unwrap_or(Duration::from_millis(5000))
.unwrap_or(Duration::from_millis(3000))
.min(Duration::from_millis(5000))
.max(Duration::from_millis(200));
let binned_range = query.covering_range()?;
@@ -439,18 +437,19 @@ pub async fn timebinned_json(
let stream = timebinned_to_collectable(stream);
let collected = Collect::new(stream, deadline, collect_max, bytes_max, None, Some(binned_range));
let collected: BoxFuture<_> = Box::pin(collected);
let collected = collected.await?;
info!("timebinned_json collected type_name {:?}", collected.type_name());
let collected = if let Some(bins) = collected
let collres = collected.await?;
info!("timebinned_json collected type_name {:?}", collres.type_name());
let collres = if let Some(bins) = collres
.as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{
info!("MATCHED");
bins.boxed_collected_with_enum_fix()
warn!("unexpected binned enum");
// bins.boxed_collected_with_enum_fix()
collres
} else {
collected
collres
};
let jsval = serde_json::to_value(&collected)?;
let jsval = serde_json::to_value(&collres)?;
Ok(jsval)
}
@@ -461,8 +460,9 @@ fn take_collector_result(coll: &mut Box<dyn items_0::collect_s::Collector>) -> O
.as_any_ref()
.downcast_ref::<items_2::binsdim0::BinsDim0CollectedResult<netpod::EnumVariant>>()
{
info!("MATCHED ENUM");
bins.boxed_collected_with_enum_fix()
warn!("unexpected binned enum");
// bins.boxed_collected_with_enum_fix()
collres
} else {
collres
};