daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Use more configured ttl

Browse Source
This commit is contained in:
Dominik Werder
2022-12-13 15:19:05 +01:00
parent b5c1039c78
commit 44364352b1
14 changed files with 2890 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
netfetch/src/insertworker.rs
View File

@@ -1,5 +1,6 @@
use crate::ca::store::DataStore;
use crate::ca::{CaConnectOpts, IngestCommons};
use crate::ca::IngestCommons;
use crate::conf::CaIngestOpts;
use crate::rt::JoinHandle;
use crate::store::{CommonInsertItemQueue, IntoSimplerError, QueryItem};
use err::Error;
@@ -52,7 +53,7 @@ pub async fn spawn_scylla_insert_workers(
pg_client: Arc<PgClient>,
store_stats: Arc<stats::CaConnStats>,
use_rate_limit_queue: bool,
opts: CaConnectOpts,
opts: CaIngestOpts,
) -> Result<Vec<JoinHandle<()>>, Error> {
let (q2_tx, q2_rx) = async_channel::bounded(insert_item_queue.receiver().capacity().unwrap_or(20000));
{
@@ -124,9 +125,9 @@ pub async fn spawn_scylla_insert_workers(
insert_item_queue.receiver()
};
let ingest_commons = ingest_commons.clone();
let ttl_msp = opts.ttl_index;
let ttl_0d = opts.ttl_d0;
let ttl_1d = opts.ttl_d1;
let ttl_msp = opts.ttl_index();
let ttl_0d = opts.ttl_d0();
let ttl_1d = opts.ttl_d1();
let fut = async move {
let backoff_0 = Duration::from_millis(10);
let mut backoff = backoff_0.clone();
@@ -140,7 +141,7 @@ pub async fn spawn_scylla_insert_workers(
};
match item {
QueryItem::ConnectionStatus(item) => {
match crate::store::insert_connection_status(item, &data_store, &stats).await {
match crate::store::insert_connection_status(item, ttl_msp, &data_store, &stats).await {
Ok(_) => {
stats.connection_status_insert_done_inc();
backoff = backoff_0;
@@ -152,7 +153,7 @@ pub async fn spawn_scylla_insert_workers(
}
}
QueryItem::ChannelStatus(item) => {
match crate::store::insert_channel_status(item, &data_store, &stats).await {
match crate::store::insert_channel_status(item, ttl_msp, &data_store, &stats).await {
Ok(_) => {
stats.channel_status_insert_done_inc();
backoff = backoff_0;
@@ -188,6 +189,7 @@ pub async fn spawn_scylla_insert_workers(
item.ts as i64,
item.ema,
item.emd,
ttl_msp.as_secs() as i32,
);
let qres = data_store.scy.execute(&data_store.qu_insert_muted, values).await;
match qres {
@@ -209,6 +211,7 @@ pub async fn spawn_scylla_insert_workers(
item.ts as i64,
item.ema,
item.emd,
ttl_msp.as_secs() as i32,
);
let qres = data_store
.scy
@@ -234,6 +237,7 @@ pub async fn spawn_scylla_insert_workers(
item.ivl,
item.interest,
item.evsize as i32,
ttl_msp.as_secs() as i32,
);
let qres = data_store.scy.execute(&data_store.qu_insert_channel_ping, params).await;
match qres {