daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Binned service can now merge from remote if no matching pre-binned grid

Browse Source
This commit is contained in:
Dominik Werder
2021-05-06 15:07:31 +02:00
parent 2f4d2ccea9
commit f391eca970
9 changed files with 121 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
disk/src/cache/pbv.rs vendored
View File

@@ -87,9 +87,8 @@ impl PreBinnedValueStream {
}
}
// TODO handle errors also here via return type.
fn setup_merged_from_remotes(&mut self) {
let g = self.query.patch.bin_t_len();
warn!("no better resolution found for g {}", g);
let evq = EventsQuery {
channel: self.query.channel.clone(),
range: self.query.patch.patch_range(),
@@ -107,8 +106,8 @@ impl PreBinnedValueStream {
let count = self.query.patch.patch_t_len() / self.query.patch.bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count).unwrap();
let s1 = MergedFromRemotes::new(evq, self.node_config.node_config.cluster.clone());
let s2 = s1.into_binned_t(range);
let s2 = s2.map(|k| {
let s1 = s1.into_binned_t(range);
let s1 = s1.map(|k| {
use MinMaxAvgScalarBinBatchStreamItem::*;
match k {
Ok(Values(k)) => Ok(PreBinnedItem::Batch(k)),
@@ -118,13 +117,13 @@ impl PreBinnedValueStream {
Err(e) => Err(e),
}
});
self.fut2 = Some(Box::pin(s2));
self.fut2 = Some(Box::pin(s1));
}
fn setup_from_higher_res_prebinned(&mut self, range: PreBinnedPatchRange) {
let g = self.query.patch.bin_t_len();
let h = range.grid_spec.bin_t_len();
info!(
trace!(
"try_setup_fetch_prebinned_higher_res found g {} h {} ratio {} mod {} {:?}",
g,
h,
@@ -172,7 +171,7 @@ impl PreBinnedValueStream {
}
fn try_setup_fetch_prebinned_higher_res(&mut self) {
info!("try_setup_fetch_prebinned_higher_res for {:?}", self.query.patch);
trace!("try_setup_fetch_prebinned_higher_res for {:?}", self.query.patch);
let range = self.query.patch.patch_range();
match PreBinnedPatchRange::covering_range(range, self.query.patch.bin_count() + 1) {
Some(range) => {

26
disk/src/cache/pbvfs.rs vendored
View File

@@ -6,6 +6,7 @@ use crate::streamlog::LogItem;
use err::Error;
use futures_core::Stream;
use futures_util::{pin_mut, FutureExt};
use http::StatusCode;
#[allow(unused_imports)]
use netpod::log::*;
use netpod::{EventDataReadStats, NodeConfigCached};
@@ -25,7 +26,6 @@ impl PreBinnedValueFetchedStream {
pub fn new(query: &PreBinnedQuery, node_config: &NodeConfigCached) -> Result<Self, Error> {
let nodeix = node_ix_for_patch(&query.patch, &query.channel, &node_config.node_config.cluster);
let node = &node_config.node_config.cluster.nodes[nodeix as usize];
warn!("TODO defining property of a PreBinnedPatchCoord? patchlen + ix? binsize + patchix? binsize + patchsize + patchix?");
let uri: hyper::Uri = format!(
"http://{}:{}/api/1/prebinned?{}",
node.host,
@@ -33,7 +33,6 @@ impl PreBinnedValueFetchedStream {
query.make_query_string()
)
.parse()?;
info!("PreBinnedValueFetchedStream open uri {}", uri);
let ret = Self {
uri,
resfut: None,
@@ -96,11 +95,23 @@ impl Stream for PreBinnedValueFetchedStream {
match resfut.poll_unpin(cx) {
Ready(res) => match res {
Ok(res) => {
info!("PreBinnedValueFetchedStream GOT result from SUB REQUEST: {:?}", res);
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1);
self.res = Some(s2);
continue 'outer;
if res.status() == StatusCode::OK {
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1);
self.res = Some(s2);
continue 'outer;
} else {
error!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
);
let e = Error::with_msg(format!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
));
self.errored = true;
Ready(Some(Err(e)))
}
}
Err(e) => {
error!("PreBinnedValueStream error in stream {:?}", e);
@@ -118,7 +129,6 @@ impl Stream for PreBinnedValueFetchedStream {
{
Ok(req) => {
let client = hyper::Client::new();
info!("PreBinnedValueFetchedStream START REQUEST FOR {:?}", req);
self.resfut = Some(client.request(req));
continue 'outer;
}