daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on file-lookup for expand-query

Browse Source
This commit is contained in:
Dominik Werder
2021-08-31 16:12:13 +02:00
parent ba568c8850
commit 6581946eaf
17 changed files with 170 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
disk/src/aggtest.rs
View File

@@ -16,7 +16,6 @@ pub fn make_test_node(id: u32) -> Node {
split: id,
ksprefix: "ks".into(),
backend: "testbackend".into(),
bin_grain_kind: 0,
archiver_appliance: None,
}
}

36
disk/src/binned.rs
View File

@@ -76,20 +76,12 @@ impl ChannelExecFunction for BinnedBinaryChannelExec {
FrameType + Framable + DeserializeOwned,
{
let _ = event_value_shape;
let range = BinnedRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
)?
.ok_or(Error::with_msg(format!(
"BinnedBinaryChannelExec BinnedRange::covering_range returned None"
)))?;
let range =
BinnedRange::covering_range(self.query.range().clone(), self.query.bin_count())?.ok_or(Error::with_msg(
format!("BinnedBinaryChannelExec BinnedRange::covering_range returned None"),
))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let souter = match PreBinnedPatchRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
) {
let souter = match PreBinnedPatchRange::covering_range(self.query.range().clone(), self.query.bin_count()) {
Ok(Some(pre_range)) => {
info!("BinnedBinaryChannelExec found pre_range: {:?}", pre_range);
if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() {
@@ -321,21 +313,13 @@ impl ChannelExecFunction for BinnedJsonChannelExec {
FrameType + Framable + DeserializeOwned,
{
let _ = event_value_shape;
let range = BinnedRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
)?
.ok_or(Error::with_msg(format!(
"BinnedJsonChannelExec BinnedRange::covering_range returned None"
)))?;
let range =
BinnedRange::covering_range(self.query.range().clone(), self.query.bin_count())?.ok_or(Error::with_msg(
format!("BinnedJsonChannelExec BinnedRange::covering_range returned None"),
))?;
let t_bin_count = range.count as u32;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let souter = match PreBinnedPatchRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
) {
let souter = match PreBinnedPatchRange::covering_range(self.query.range().clone(), self.query.bin_count()) {
Ok(Some(pre_range)) => {
info!("BinnedJsonChannelExec found pre_range: {:?}", pre_range);
if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() {

8
disk/src/binned/binnedfrompbv.rs
View File

@@ -101,14 +101,12 @@ where
self.res = Some(s2);
continue 'outer;
} else {
error!(
let msg = format!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
);
let e = Error::with_msg(format!(
"PreBinnedValueFetchedStream got non-OK result from sub request: {:?}",
res
));
error!("{}", msg);
let e = Error::with_msg_no_trace(msg);
self.errored = true;
Ready(Some(Err(e)))
}

8
disk/src/binned/pbv.rs
View File

@@ -129,7 +129,7 @@ where
}
// TODO do I need to set up more transformations or binning to deliver the requested data?
let count = self.query.patch().patch_t_len() / self.query.patch().bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count as u32, self.node_config.node.bin_grain_kind)?
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
@@ -209,11 +209,7 @@ where
fn try_setup_fetch_prebinned_higher_res(&mut self) -> Result<(), Error> {
let range = self.query.patch().patch_range();
match PreBinnedPatchRange::covering_range(
range,
self.query.patch().bin_count() + 1,
self.node_config.node.bin_grain_kind,
) {
match PreBinnedPatchRange::covering_range(range, self.query.patch().bin_count() + 1) {
Ok(Some(range)) => {
self.fut2 = Some(self.setup_from_higher_res_prebinned(range)?);
}

58
disk/src/dataopen.rs
View File

@@ -3,6 +3,7 @@ use bytes::BytesMut;
use err::Error;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::timeunits::DAY;
use netpod::{ChannelConfig, NanoRange, Nanos, Node};
use std::path::PathBuf;
use std::time::Instant;
@@ -162,3 +163,60 @@ async fn open_files_inner(
// TODO keep track of number of running
Ok(())
}
/**
Try to find and position the file with the youngest event before the requested range.
*/
async fn single_file_before_range(
chtx: async_channel::Sender<Result<OpenedFile, Error>>,
range: NanoRange,
channel_config: ChannelConfig,
node: Node,
) -> Result<(), Error> {
Ok(())
}
#[test]
fn single_file() {
let range = netpod::NanoRange { beg: 0, end: 0 };
let chn = netpod::Channel {
backend: "testbackend".into(),
name: "scalar-i32-be".into(),
};
let cfg = ChannelConfig {
channel: chn,
keyspace: 2,
time_bin_size: Nanos { ns: DAY },
scalar_type: netpod::ScalarType::I32,
compression: false,
shape: netpod::Shape::Scalar,
array: false,
byte_order: netpod::ByteOrder::big_endian(),
};
let cluster = taskrun::test_cluster();
let task = async move {
let (tx, rx) = async_channel::bounded(2);
let jh = taskrun::spawn(single_file_before_range(tx, range, cfg, cluster.nodes[0].clone()));
loop {
match rx.recv().await {
Ok(k) => match k {
Ok(k) => {
info!("opened file: {:?}", k.path);
}
Err(e) => {
error!("error while trying to open {:?}", e);
break;
}
},
Err(e) => {
// channel closed.
info!("channel closed");
break;
}
}
}
jh.await??;
Ok(())
};
taskrun::run(task).unwrap();
}

1
disk/src/gen.rs
View File

@@ -81,7 +81,6 @@ pub async fn gen_test_data() -> Result<(), Error> {
cache_base_path: data_base_path.join(format!("node{:02}", i1)),
ksprefix: ksprefix.clone(),
backend: "testbackend".into(),
bin_grain_kind: 0,
archiver_appliance: None,
};
ensemble.nodes.push(node);

2
disk/src/lib.rs
View File

@@ -308,7 +308,7 @@ pub struct FileChunkRead {
}
pub fn file_content_stream(
mut file: tokio::fs::File,
mut file: File,
buffer_size: usize,
) -> impl Stream<Item = Result<FileChunkRead, Error>> + Send {
async_stream::stream! {