WIP on binning for archeng

This commit is contained in:
Dominik Werder
2021-11-10 17:25:40 +01:00
parent 2f608a8a4e
commit 21e16cfa6d
18 changed files with 504 additions and 142 deletions

View File

@@ -126,7 +126,12 @@ where
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone()).map(|k| {
info!(
"setup_merged_from_remotes, MergedFromRemotes yields {:?}",
show_event_basic_info(&k)
);
});
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(
s,
range,

View File

@@ -176,7 +176,10 @@ where
let path = cfd.path(&node_config);
let enc = serde_cbor::to_vec(&values)?;
let ts1 = Instant::now();
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
tokio::fs::create_dir_all(path.parent().unwrap()).await.map_err(|e| {
error!("can not create cache directory {:?}", path.parent());
e
})?;
let now = Utc::now();
let mut h = crc32fast::Hasher::new();
h.update(&now.timestamp_nanos().to_le_bytes());

View File

@@ -64,10 +64,7 @@ where
}
},
Err(e) => {
error!(
"EventsFromFrames ~~~~~~~~ ERROR on frame payload {}",
frame.buf().len(),
);
error!("frame payload len {} tyid {} {}", frame.buf().len(), frame.tyid(), e);
self.errored = true;
Ready(Some(Err(e)))
}