daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Distinct sub-query type

Browse Source
This commit is contained in:
Dominik Werder
2023-06-22 21:10:58 +02:00
parent d9ac27cd75
commit 0260f4b4d6
20 changed files with 374 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
disk/src/aggtest.rs
View File

@@ -9,13 +9,13 @@ use netpod::timeunits::*;
use netpod::ByteOrder;
use netpod::ByteSize;
use netpod::DiskIoTune;
use netpod::DtNano;
use netpod::Node;
use netpod::ScalarType;
use netpod::SfChFetchInfo;
use netpod::SfDatabuffer;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
pub fn make_test_node(id: u32) -> Node {
Node {
@@ -51,7 +51,7 @@ async fn agg_x_dim_0_inner() {
channel_config: SfDbChConf {
channel: SfDbChannel::from_name("sf-databuffer", "S10BC01-DBAM070:EOM1_T1"),
keyspace: 2,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
array: false,
shape: Shape::Scalar,
scalar_type: ScalarType::F64,
@@ -66,13 +66,13 @@ async fn agg_x_dim_0_inner() {
"sf-databuffer",
"S10BC01-DBAM070:EOM1_T1",
2,
TsNano(DAY),
DtNano::from_ns(DAY),
ByteOrder::Big,
ScalarType::F64,
Shape::Scalar,
);
let _bin_count = 20;
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.0;
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.ns();
let ts2 = ts1 + HOUR * 24;
let range = NanoRange { beg: ts1, end: ts2 };
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
@@ -113,7 +113,7 @@ async fn agg_x_dim_1_inner() {
channel_config: SfDbChConf {
channel: SfDbChannel::from_name("ks", "wave1"),
keyspace: 3,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
array: true,
shape: Shape::Wave(1024),
scalar_type: ScalarType::F64,
@@ -128,13 +128,13 @@ async fn agg_x_dim_1_inner() {
"ks",
"wave1",
2,
TsNano(DAY),
DtNano::from_ns(DAY),
ByteOrder::Big,
ScalarType::F64,
Shape::Scalar,
);
let _bin_count = 10;
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.0;
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.ns();
let ts2 = ts1 + HOUR * 24;
let range = NanoRange { beg: ts1, end: ts2 };
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));

4
disk/src/disk.rs
View File

@@ -28,12 +28,12 @@ use futures_util::TryFutureExt;
use netpod::log::*;
use netpod::ByteOrder;
use netpod::DiskIoTune;
use netpod::DtNano;
use netpod::Node;
use netpod::ReadSys;
use netpod::ScalarType;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
use serde::Deserialize;
use serde::Serialize;
use std::collections::VecDeque;
@@ -65,7 +65,7 @@ use tokio::sync::mpsc;
pub struct SfDbChConf {
pub channel: SfDbChannel,
pub keyspace: u8,
pub time_bin_size: TsNano,
pub time_bin_size: DtNano,
pub scalar_type: ScalarType,
pub compression: bool,
pub shape: Shape,

11
disk/src/gen.rs
View File

@@ -7,6 +7,7 @@ use err::Error;
use netpod::log::*;
use netpod::timeunits::*;
use netpod::ByteOrder;
use netpod::DtNano;
use netpod::GenVar;
use netpod::Node;
use netpod::ScalarType;
@@ -36,7 +37,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
config: SfDbChConf {
channel: SfDbChannel::from_name(&backend, "scalar-i32-be"),
keyspace: 2,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
scalar_type: ScalarType::I32,
byte_order: ByteOrder::Big,
shape: Shape::Scalar,
@@ -51,7 +52,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
config: SfDbChConf {
channel: SfDbChannel::from_name(&backend, "wave-f64-be-n21"),
keyspace: 3,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
array: true,
scalar_type: ScalarType::F64,
shape: Shape::Wave(21),
@@ -66,7 +67,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
config: SfDbChConf {
channel: SfDbChannel::from_name(&backend, "wave-u16-le-n77"),
keyspace: 3,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
scalar_type: ScalarType::U16,
byte_order: ByteOrder::Little,
shape: Shape::Wave(77),
@@ -81,7 +82,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
config: SfDbChConf {
channel: SfDbChannel::from_name(&backend, "tw-scalar-i32-be"),
keyspace: 2,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
scalar_type: ScalarType::I32,
byte_order: ByteOrder::Little,
shape: Shape::Scalar,
@@ -96,7 +97,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
config: SfDbChConf {
channel: SfDbChannel::from_name(&backend, "const-regular-scalar-i32-be"),
keyspace: 2,
time_bin_size: TsNano(DAY),
time_bin_size: DtNano::from_ns(DAY),
scalar_type: ScalarType::I32,
byte_order: ByteOrder::Little,
shape: Shape::Scalar,

14
disk/src/merge/mergedblobsfromremotes.rs
View File

@@ -9,6 +9,9 @@ use netpod::log::*;
use netpod::ChannelTypeConfigGen;
use netpod::Cluster;
use netpod::PerfOpts;
use query::api4::events::EventsSubQuery;
use query::api4::events::EventsSubQuerySelect;
use query::api4::events::EventsSubQuerySettings;
use query::api4::events::PlainEventsQuery;
use std::future::Future;
use std::pin::Pin;
@@ -30,10 +33,17 @@ pub struct MergedBlobsFromRemotes {
impl MergedBlobsFromRemotes {
pub fn new(evq: PlainEventsQuery, perf_opts: PerfOpts, ch_conf: ChannelTypeConfigGen, cluster: Cluster) -> Self {
debug!("MergedBlobsFromRemotes evq {:?}", evq);
let select = EventsSubQuerySelect::new(ch_conf.clone(), evq.range().clone(), evq.transform().clone());
let settings = EventsSubQuerySettings::from(&evq);
let subq = EventsSubQuery::from_parts(select, settings);
let mut tcp_establish_futs = Vec::new();
for node in &cluster.nodes {
let f =
x_processed_event_blobs_stream_from_node(evq.clone(), ch_conf.clone(), perf_opts.clone(), node.clone());
let f = x_processed_event_blobs_stream_from_node(
subq.clone(),
ch_conf.clone(),
perf_opts.clone(),
node.clone(),
);
let f: T002<EventFull> = Box::pin(f);
tcp_establish_futs.push(f);
}

36
disk/src/raw/conn.rs
View File

@@ -18,7 +18,7 @@ use netpod::ByteSize;
use netpod::DiskIoTune;
use netpod::NodeConfigCached;
use netpod::SfChFetchInfo;
use query::api4::events::PlainEventsQuery;
use query::api4::events::EventsSubQuery;
use std::pin::Pin;
const TEST_BACKEND: &str = "testbackend-00";
@@ -53,15 +53,16 @@ fn make_num_pipeline_stream_evs(
}
pub async fn make_event_pipe(
evq: &PlainEventsQuery,
evq: EventsSubQuery,
fetch_info: SfChFetchInfo,
ncc: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
// sf-databuffer type backends identify channels by their (backend, name) only.
let range = evq.range().clone();
let one_before = evq.transform().need_one_before_range();
info!(
"make_event_pipe need_expand {need_expand} {evq:?}",
need_expand = evq.one_before_range()
need_expand = one_before
);
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
// TODO should not need this for correctness.
@@ -71,6 +72,7 @@ pub async fn make_event_pipe(
} else {
128
};
let do_decompress = true;
let event_blobs = EventChunkerMultifile::new(
(&range).try_into()?,
fetch_info.clone(),
@@ -78,8 +80,8 @@ pub async fn make_event_pipe(
ncc.ix,
DiskIoTune::default(),
event_chunker_conf,
evq.one_before_range(),
true,
one_before,
do_decompress,
out_max_len,
);
error!("TODO replace AggKind in the called code");
@@ -155,18 +157,18 @@ pub fn make_remote_event_blobs_stream(
}
pub async fn make_event_blobs_pipe_real(
evq: &PlainEventsQuery,
subq: &EventsSubQuery,
fetch_info: &SfChFetchInfo,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
if false {
match dbconn::channel_exists(evq.channel(), &node_config).await {
match dbconn::channel_exists(subq.name(), &node_config).await {
Ok(_) => (),
Err(e) => return Err(e)?,
}
}
let expand = evq.one_before_range();
let range = evq.range();
let expand = subq.transform().need_one_before_range();
let range = subq.range();
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
// TODO should depend on host config
let do_local = node_config.node_config.cluster.is_central_storage;
@@ -204,14 +206,14 @@ pub async fn make_event_blobs_pipe_real(
}
pub async fn make_event_blobs_pipe_test(
evq: &PlainEventsQuery,
subq: &EventsSubQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
warn!("GENERATE INMEM TEST DATA");
let node_count = node_config.node_config.cluster.nodes.len() as u64;
let node_ix = node_config.ix as u64;
let chn = evq.channel().name();
let range = evq.range().clone();
let chn = subq.name();
let range = subq.range().clone();
if chn == "test-gen-i32-dim0-v00" {
Ok(Box::pin(EventBlobsGeneratorI32Test00::new(node_ix, node_count, range)))
} else if chn == "test-gen-i32-dim0-v01" {
@@ -247,14 +249,14 @@ pub async fn make_event_blobs_pipe_test(
}
pub async fn make_event_blobs_pipe(
evq: &PlainEventsQuery,
subq: &EventsSubQuery,
fetch_info: &SfChFetchInfo,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
debug!("make_event_blobs_pipe {evq:?}");
if evq.channel().backend() == TEST_BACKEND {
make_event_blobs_pipe_test(evq, node_config).await
debug!("make_event_blobs_pipe {subq:?}");
if subq.backend() == TEST_BACKEND {
make_event_blobs_pipe_test(subq, node_config).await
} else {
make_event_blobs_pipe_real(evq, fetch_info, node_config).await
make_event_blobs_pipe_real(subq, fetch_info, node_config).await
}
}