daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adjust caching and disk usage log output

Browse Source
This commit is contained in:
Dominik Werder
2021-05-10 13:56:11 +02:00
parent 2df0be8ed2
commit 8cedf06d9c
15 changed files with 128 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
retrieval/src/bin/retrieval.rs
View File

@@ -53,6 +53,7 @@ async fn go() -> Result<(), Error> {
end,
opts.bins,
cache_usage,
opts.disk_stats_every_kb,
)
.await?;
}

2
retrieval/src/cli.rs
View File

@@ -53,4 +53,6 @@ pub struct BinnedClient {
pub ignore_cache: bool,
#[clap(long)]
pub recreate_cache: bool,
#[clap(long, default_value = "1048576")]
pub disk_stats_every_kb: u32,
}

4
retrieval/src/client.rs
View File

@@ -19,12 +19,13 @@ pub async fn get_binned(
end_date: DateTime<Utc>,
bin_count: u32,
cache_usage: CacheUsage,
disk_stats_every_kb: u32,
) -> Result<(), Error> {
info!("------- get_binned client");
let t1 = Utc::now();
let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
let uri = format!(
"http://{}:{}/api/1/binned?channel_backend={}&channel_name={}&beg_date={}&end_date={}&bin_count={}&cache_usage={}",
"http://{}:{}/api/1/binned?channel_backend={}&channel_name={}&beg_date={}&end_date={}&bin_count={}&cache_usage={}&disk_stats_every_kb={}",
host,
port,
channel_backend,
@@ -33,6 +34,7 @@ pub async fn get_binned(
end_date.format(date_fmt),
bin_count,
cache_usage.query_param_value(),
disk_stats_every_kb,
);
info!("get_binned uri {:?}", uri);
let req = hyper::Request::builder()

6
retrieval/src/test.rs
View File

@@ -10,7 +10,7 @@ use futures_util::TryStreamExt;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::{Cluster, Database, Node, PerfOpts};
use netpod::{ByteSize, Cluster, Database, Node, PerfOpts};
use std::future::ready;
use tokio::io::AsyncRead;
@@ -94,9 +94,10 @@ where
let channel_backend = "testbackend";
let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let disk_stats_every = ByteSize::kb(1024);
// TODO have a function to form the uri, including perf opts:
let uri = format!(
"http://{}:{}/api/1/binned?cache_usage=ignore&channel_backend={}&channel_name={}&bin_count={}&beg_date={}&end_date={}",
"http://{}:{}/api/1/binned?cache_usage=ignore&channel_backend={}&channel_name={}&bin_count={}&beg_date={}&end_date={}&disk_stats_every_kb={}",
node0.host,
node0.port,
channel_backend,
@@ -104,6 +105,7 @@ where
bin_count,
beg_date.format(date_fmt),
end_date.format(date_fmt),
disk_stats_every.bytes() / 1024,
);
info!("get_binned_channel get {}", uri);
let req = hyper::Request::builder()