daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on sf databuffer channel config decision

Browse Source
This commit is contained in:
Dominik Werder
2023-06-23 16:31:55 +02:00
parent 0260f4b4d6
commit 64d2996c48
26 changed files with 249 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
disk/src/aggtest.rs
View File

@@ -75,7 +75,7 @@ async fn agg_x_dim_0_inner() {
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.ns();
let ts2 = ts1 + HOUR * 24;
let range = NanoRange { beg: ts1, end: ts2 };
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
// TODO let upstream already provide DiskIoTune:
let mut disk_io_tune = DiskIoTune::default_for_testing();
disk_io_tune.read_buffer_len = query.buffer_size as usize;
@@ -137,7 +137,7 @@ async fn agg_x_dim_1_inner() {
let ts1 = query.timebin as u64 * query.channel_config.time_bin_size.ns();
let ts2 = ts1 + HOUR * 24;
let range = NanoRange { beg: ts1, end: ts2 };
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
// TODO let upstream already provide DiskIoTune:
let mut disk_io_tune = DiskIoTune::default_for_testing();
disk_io_tune.read_buffer_len = query.buffer_size as usize;

2
disk/src/channelconfig.rs
View File

@@ -24,7 +24,7 @@ pub async fn config_entry_best_match(
}
}
pub async fn configs(channel: SfDbChannel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
pub async fn channel_configs(channel: SfDbChannel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
read_local_config(channel.clone(), node_config.clone()).await
}

19
disk/src/merge/mergedblobsfromremotes.rs
View File

@@ -6,13 +6,8 @@ use items_0::streamitem::Sitemty;
use items_2::eventfull::EventFull;
use items_2::merger::Merger;
use netpod::log::*;
use netpod::ChannelTypeConfigGen;
use netpod::Cluster;
use netpod::PerfOpts;
use query::api4::events::EventsSubQuery;
use query::api4::events::EventsSubQuerySelect;
use query::api4::events::EventsSubQuerySettings;
use query::api4::events::PlainEventsQuery;
use std::future::Future;
use std::pin::Pin;
use std::task::Context;
@@ -31,19 +26,11 @@ pub struct MergedBlobsFromRemotes {
}
impl MergedBlobsFromRemotes {
pub fn new(evq: PlainEventsQuery, perf_opts: PerfOpts, ch_conf: ChannelTypeConfigGen, cluster: Cluster) -> Self {
debug!("MergedBlobsFromRemotes evq {:?}", evq);
let select = EventsSubQuerySelect::new(ch_conf.clone(), evq.range().clone(), evq.transform().clone());
let settings = EventsSubQuerySettings::from(&evq);
let subq = EventsSubQuery::from_parts(select, settings);
pub fn new(subq: EventsSubQuery, cluster: Cluster) -> Self {
debug!("MergedBlobsFromRemotes evq {:?}", subq);
let mut tcp_establish_futs = Vec::new();
for node in &cluster.nodes {
let f = x_processed_event_blobs_stream_from_node(
subq.clone(),
ch_conf.clone(),
perf_opts.clone(),
node.clone(),
);
let f = x_processed_event_blobs_stream_from_node(subq.clone(), node.clone());
let f: T002<EventFull> = Box::pin(f);
tcp_establish_futs.push(f);
}

4
disk/src/raw/conn.rs
View File

@@ -64,7 +64,7 @@ pub async fn make_event_pipe(
"make_event_pipe need_expand {need_expand} {evq:?}",
need_expand = one_before
);
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
// TODO should not need this for correctness.
// Should limit based on return size and latency.
let out_max_len = if ncc.node_config.cluster.is_central_storage {
@@ -169,7 +169,7 @@ pub async fn make_event_blobs_pipe_real(
}
let expand = subq.transform().need_one_before_range();
let range = subq.range();
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
// TODO should depend on host config
let do_local = node_config.node_config.cluster.is_central_storage;
let pipe = if do_local {