daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2021-04-27 11:41:33 +02:00
parent bd9c231310
commit 0b40702b6c
12 changed files with 170 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
disk/Cargo.toml
View File

@@ -33,3 +33,4 @@ err = { path = "../err" }
taskrun = { path = "../taskrun" }
netpod = { path = "../netpod" }
bitshuffle = { path = "../bitshuffle" }
dbconn = { path = "../dbconn" }

30
disk/src/cache.rs
View File

@@ -1,6 +1,7 @@
use crate::agg::MinMaxAvgScalarEventBatch;
use crate::binnedstream::BinnedStream;
use crate::cache::pbv::PreBinnedValueByteStream;
use crate::channelconfig::{extract_matching_config_entry, read_local_config};
use crate::frame::makeframe::make_frame;
use crate::merge::MergedMinMaxAvgScalarStream;
use crate::raw::EventsQuery;
@@ -68,33 +69,10 @@ pub async fn binned_bytes_for_http(
node_config: Arc<NodeConfig>,
query: &Query,
) -> Result<BinnedBytesForHttpStream, Error> {
let channel_config = super::channelconfig::read_local_config(&query.channel, node_config.clone()).await?;
let entry;
{
let mut ixs = vec![];
for i1 in 0..channel_config.entries.len() {
let e1 = &channel_config.entries[i1];
if i1 + 1 < channel_config.entries.len() {
let e2 = &channel_config.entries[i1 + 1];
if e1.ts < query.range.end && e2.ts >= query.range.beg {
ixs.push(i1);
}
} else {
if e1.ts < query.range.end {
ixs.push(i1);
}
}
}
if ixs.len() == 0 {
return Err(Error::with_msg(format!("no config entries found")));
} else if ixs.len() > 1 {
return Err(Error::with_msg(format!("too many config entries found: {}", ixs.len())));
}
entry = &channel_config.entries[ixs[0]];
}
let range = &query.range;
let channel_config = read_local_config(&query.channel, node_config.clone()).await?;
let entry = extract_matching_config_entry(range, &channel_config);
info!("found config entry {:?}", entry);
let grid = PreBinnedPatchRange::covering_range(query.range.clone(), query.count);
match grid {
Some(spec) => {

28
disk/src/channelconfig.rs
View File

@@ -1,5 +1,5 @@
use err::Error;
use netpod::{Channel, NodeConfig};
use netpod::{Channel, NanoRange, NodeConfig};
use nom::number::complete::{be_i16, be_i32, be_i64, be_i8, be_u8};
use nom::Needed;
#[allow(unused_imports)]
@@ -278,6 +278,32 @@ pub async fn read_local_config(channel: &Channel, node_config: Arc<NodeConfig>)
Ok(config.1)
}
pub fn extract_matching_config_entry<'a>(
range: &NanoRange,
channel_config: &'a Config,
) -> Result<&'a ConfigEntry, Error> {
let mut ixs = vec![];
for i1 in 0..channel_config.entries.len() {
let e1 = &channel_config.entries[i1];
if i1 + 1 < channel_config.entries.len() {
let e2 = &channel_config.entries[i1 + 1];
if e1.ts < range.end && e2.ts >= range.beg {
ixs.push(i1);
}
} else {
if e1.ts < range.end {
ixs.push(i1);
}
}
}
if ixs.len() == 0 {
return Err(Error::with_msg(format!("no config entries found")));
} else if ixs.len() > 1 {
return Err(Error::with_msg(format!("too many config entries found: {}", ixs.len())));
}
Ok(&channel_config.entries[ixs[0]])
}
#[cfg(test)]
mod test {
use super::parse_config;

4
disk/src/gen.rs
View File

@@ -32,11 +32,11 @@ pub async fn gen_test_data() -> Result<(), Error> {
big_endian: true,
compression: true,
},
time_spacing: MS * 2000,
time_spacing: MS * 1000,
};
ensemble.channels.push(chn);
}
for i1 in 0..13 {
for i1 in 0..3 {
let node = Node {
id: i1,
host: "localhost".into(),

56
disk/src/raw/conn.rs
View File

@@ -1,4 +1,5 @@
use crate::agg::{IntoBinnedXBins1, IntoDim1F32Stream, MinMaxAvgScalarEventBatch};
use crate::channelconfig::{extract_matching_config_entry, read_local_config};
use crate::eventblobs::EventBlobsComplete;
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::{make_frame, make_term_frame};
@@ -123,6 +124,10 @@ async fn raw_conn_handler_inner_try(
return Err((Error::with_msg("can not parse request json"), netout))?;
}
};
match dbconn::channel_exists(&evq.channel, node_config.clone()).await {
Ok(_) => (),
Err(e) => return Err((e, netout))?,
}
debug!(
"\n\nREQUEST FOR RANGE {} {}\n\n",
evq.range.beg / SEC,
@@ -132,19 +137,48 @@ async fn raw_conn_handler_inner_try(
"TODO decide on response content based on the parsed json query\n{:?}",
evq
);
let range = &evq.range;
let channel_config = match read_local_config(&evq.channel, node_config.clone()).await {
Ok(k) => k,
Err(e) => return Err((e, netout))?,
};
let entry = extract_matching_config_entry(range, &channel_config)?;
info!("found config entry {:?}", entry);
let shape = match &entry.shape {
Some(lens) => {
if lens.len() == 1 {
Shape::Wave(lens[0])
} else {
return Err(Error::with_msg(format!("Channel config unsupported shape {:?}", entry)))?;
}
}
None => Shape::Scalar,
};
/*
TODO
This endpoint should deliver events over some time range, across files.
Therefore, based on the query and the found channel config, list the available files in the
candidate directories, and iterate over events from those files.
!!! use index if available
• Generate index file for my test data.
• Use index file if available.
• If not, must use binary search if possible in that type.
Create a new type in place of AggQuerySingleChannel?
*/
err::todoval();
let query = netpod::AggQuerySingleChannel {
channel_config: netpod::ChannelConfig {
channel: netpod::Channel {
backend: "test1".into(),
name: "wave1".into(),
},
keyspace: 3,
time_bin_size: DAY,
shape: Shape::Wave(17),
scalar_type: ScalarType::F64,
big_endian: true,
array: true,
compression: true,
channel: evq.channel.clone(),
keyspace: entry.ks as u8,
time_bin_size: entry.bs as u64,
shape: shape,
scalar_type: ScalarType::from_dtype_index(entry.dtype.to_i16() as u8),
big_endian: entry.is_big_endian,
array: entry.is_array,
compression: entry.is_compressed,
},
// TODO use a NanoRange and search for matching files
timebin: 0,