Ingest swissfel ca until before electric test

netfetch/src/ca.rs

@@ -199,6 +199,11 @@ pub async fn ca_connect(opts: CaIngestOpts, channels: &Vec<String>) -> Result<()
 
     // TODO use a new stats type:
     let store_stats = Arc::new(CaConnStats::new());
+    let ttls = crate::insertworker::Ttls {
+        index: opts.ttl_index(),
+        d0: opts.ttl_d0(),
+        d1: opts.ttl_d1(),
+    };
     let jh_insert_workers = spawn_scylla_insert_workers(
         opts.scylla().clone(),
         opts.insert_scylla_sessions(),
@@ -208,7 +213,7 @@ pub async fn ca_connect(opts: CaIngestOpts, channels: &Vec<String>) -> Result<()
         pg_client.clone(),
         store_stats.clone(),
         opts.use_rate_limit_queue(),
-        opts.clone(),
+        ttls,
     )
     .await?;
 

netfetch/src/ca/findioc.rs

@@ -45,7 +45,6 @@ struct SearchBatch {
 #[derive(Debug)]
 pub struct FindIocRes {
     pub channel: String,
-    pub query_addr: Option<SocketAddrV4>,
     pub response_addr: Option<SocketAddrV4>,
     pub addr: Option<SocketAddrV4>,
     pub dt: Duration,
@@ -392,8 +391,6 @@ impl FindIocStream {
                                             let dt = tsnow.saturating_duration_since(batch.ts_beg);
                                             let res = FindIocRes {
                                                 channel: ch.into(),
-                                                // TODO associate a batch with a specific query address.
-                                                query_addr: None,
                                                 response_addr: Some(src.clone()),
                                                 addr: Some(addr),
                                                 dt,
@@ -463,7 +460,6 @@ impl FindIocStream {
         }
         for ((sid, ch), dt) in sids.into_iter().zip(chns).zip(dts) {
             let res = FindIocRes {
-                query_addr: None,
                 response_addr: None,
                 channel: ch,
                 addr: None,

netfetch/src/ca/search.rs

@@ -60,8 +60,8 @@ pub async fn ca_search(opts: CaIngestOpts, channels: &Vec<String>) -> Result<(),
         const TEXT: tokio_postgres::types::Type = tokio_postgres::types::Type::TEXT;
         pg_client
             .prepare_typed(
-                "insert into ioc_by_channel_log (facility, channel, queryaddr, responseaddr, addr) values ($1, $2, $3, $4, $5)",
-                &[TEXT, TEXT, TEXT, TEXT, TEXT],
+                "insert into ioc_by_channel_log (facility, channel, responseaddr, addr) values ($1, $2, $3, $4)",
+                &[TEXT, TEXT, TEXT, TEXT],
             )
             .await
             .unwrap()
@@ -154,14 +154,10 @@ pub async fn ca_search(opts: CaIngestOpts, channels: &Vec<String>) -> Result<(),
             if do_block {
                 info!("blacklisting {item:?}");
             } else {
-                let queryaddr = item.query_addr.map(|x| x.to_string());
                 let responseaddr = item.response_addr.map(|x| x.to_string());
                 let addr = item.addr.map(|x| x.to_string());
                 pg_client
-                    .execute(
-                        &qu_insert,
-                        &[&opts.backend(), &item.channel, &queryaddr, &responseaddr, &addr],
-                    )
+                    .execute(&qu_insert, &[&opts.backend(), &item.channel, &responseaddr, &addr])
                     .await
                     .unwrap();
             }

netfetch/src/insertworker.rs

@@ -1,6 +1,5 @@
 use crate::ca::store::DataStore;
 use crate::ca::IngestCommons;
-use crate::conf::CaIngestOpts;
 use crate::rt::JoinHandle;
 use crate::store::{CommonInsertItemQueue, IntoSimplerError, QueryItem};
 use err::Error;
@@ -44,6 +43,12 @@ async fn back_off_sleep(backoff_dt: &mut Duration) {
     tokio::time::sleep(*backoff_dt).await;
 }
 
+pub struct Ttls {
+    pub index: Duration,
+    pub d0: Duration,
+    pub d1: Duration,
+}
+
 pub async fn spawn_scylla_insert_workers(
     scyconf: ScyllaConfig,
     insert_scylla_sessions: usize,
@@ -53,7 +58,7 @@ pub async fn spawn_scylla_insert_workers(
     pg_client: Arc<PgClient>,
     store_stats: Arc<stats::CaConnStats>,
     use_rate_limit_queue: bool,
-    opts: CaIngestOpts,
+    ttls: Ttls,
 ) -> Result<Vec<JoinHandle<()>>, Error> {
     let (q2_tx, q2_rx) = async_channel::bounded(insert_item_queue.receiver().capacity().unwrap_or(20000));
     {
@@ -125,9 +130,9 @@ pub async fn spawn_scylla_insert_workers(
             insert_item_queue.receiver()
         };
         let ingest_commons = ingest_commons.clone();
-        let ttl_msp = opts.ttl_index();
-        let ttl_0d = opts.ttl_d0();
-        let ttl_1d = opts.ttl_d1();
+        let ttl_msp = ttls.index;
+        let ttl_0d = ttls.d0;
+        let ttl_1d = ttls.d1;
         let fut = async move {
             let backoff_0 = Duration::from_millis(10);
             let mut backoff = backoff_0.clone();