Update dependencies, refactor error type

daqingest/Cargo.toml

@@ -13,19 +13,19 @@ path = "src/bin/daqingest.rs"
 
 [dependencies]
 clap = { version = "4.0.22", features = ["derive", "cargo"] }
-tokio = { version = "1.23.0", features = ["rt-multi-thread", "io-util", "net", "time", "sync", "fs", "tracing"] }
+tokio = { version = "1.32.0", features = ["rt-multi-thread", "io-util", "net", "time", "sync", "fs", "tracing"] }
 tracing = "0.1.37"
 futures-util = "0.3"
-async-channel = "1.6"
+async-channel = "1.9.0"
 chrono = "0.4"
 bytes = "1.4.0"
-scylla = "0.8.1"
+scylla = "0.9.0"
 tokio-postgres = "0.7.7"
 serde = { version = "1.0", features = ["derive"] }
 libc = "0.2"
-err = { path = "../../daqbuffer/err" }
+err = { path = "../../daqbuffer/crates/err" }
+netpod = { path = "../../daqbuffer/crates/netpod" }
+taskrun = { path = "../../daqbuffer/crates/taskrun" }
 log = { path = "../log" }
-netpod = { path = "../../daqbuffer/netpod" }
 stats = { path = "../stats" }
 netfetch = { path = "../netfetch" }
-taskrun = { path = "../../daqbuffer/taskrun" }

daqingest/src/opts.rs

@@ -1,8 +1,7 @@
-use std::net::SocketAddr;
-
 use clap::ArgAction::Count;
 use clap::Parser;
 use netfetch::zmtp::ZmtpClientOpts;
+use std::net::SocketAddr;
 
 #[derive(Debug, Parser)]
 #[command(author, version, about)]

netfetch/Cargo.toml

@@ -9,15 +9,15 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_cbor = "0.11"
 serde_yaml = "0.9.16"
-tokio = { version = "1.28.2", features = ["rt-multi-thread", "io-util", "net", "time", "sync", "fs", "tracing"] }
-tokio-stream = { version = "0.1", features = ["fs"]}
+tokio = { version = "1.32.0", features = ["rt-multi-thread", "io-util", "net", "time", "sync", "fs", "tracing"] }
+tokio-stream = { version = "0.1", features = ["fs"] }
 tracing = "0.1.37"
-async-channel = "1.8"
+async-channel = "1.9.0"
 bytes = "1.4"
 arrayref = "0.3"
 byteorder = "1.4"
 futures-util = "0.3"
-scylla = "0.8.1"
+scylla = "0.9.0"
 tokio-postgres = "0.7.8"
 md-5 = "0.10"
 hex = "0.4"
@@ -32,13 +32,12 @@ humantime = "2.1"
 humantime-serde = "1.1"
 pin-project = "1"
 lazy_static = "1"
-thiserror = "1"
 log = { path = "../log" }
 stats = { path = "../stats" }
-err = { path = "../../daqbuffer/err" }
-netpod = { path = "../../daqbuffer/netpod" }
-items_0 = { path = "../../daqbuffer/items_0" }
-items_2 = { path = "../../daqbuffer/items_2" }
-streams = { path = "../../daqbuffer/streams" }
-taskrun = { path = "../../daqbuffer/taskrun" }
-bitshuffle = { path = "../../daqbuffer/bitshuffle" }
+err = { path = "../../daqbuffer/crates/err" }
+netpod = { path = "../../daqbuffer/crates/netpod" }
+items_0 = { path = "../../daqbuffer/crates/items_0" }
+items_2 = { path = "../../daqbuffer/crates/items_2" }
+streams = { path = "../../daqbuffer/crates/streams" }
+taskrun = { path = "../../daqbuffer/crates/taskrun" }
+bitshuffle = { path = "../../daqbuffer/crates/bitshuffle" }

netfetch/src/bsreadclient.rs

@@ -18,6 +18,8 @@ use crate::zmtp::zmtpproto::ZmtpMessage;
 use crate::zmtp::ZmtpClientOpts;
 use crate::zmtp::ZmtpEvent;
 use async_channel::Sender;
+use err::thiserror;
+use err::ThisError;
 use futures_util::StreamExt;
 use netpod::log::*;
 use netpod::timeunits::HOUR;
@@ -34,7 +36,7 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::time::Instant;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, ThisError)]
 pub enum Error {
     #[error("InsertQueueSenderMissing")]
     InsertQueueSenderMissing,

netfetch/src/ca/proto.rs

@@ -1,5 +1,7 @@
 use crate::netbuf;
 use crate::netbuf::NetBuf;
+use err::thiserror;
+use err::ThisError;
 use futures_util::pin_mut;
 use futures_util::Stream;
 use log::*;
@@ -18,7 +20,7 @@ use tokio::io::AsyncWrite;
 use tokio::io::ReadBuf;
 use tokio::net::TcpStream;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, ThisError)]
 pub enum Error {
     #[error("{0}")]
     NetBuf(#[from] netbuf::Error),

netfetch/src/ca/store.rs

@@ -1,12 +1,20 @@
-use err::Error;
-use futures_util::StreamExt;
+use err::thiserror;
+use err::ThisError;
 use netpod::ScyllaConfig;
 use scylla::execution_profile::ExecutionProfileBuilder;
 use scylla::prepared_statement::PreparedStatement;
 use scylla::statement::Consistency;
+use scylla::transport::errors::NewSessionError;
+use scylla::transport::errors::QueryError;
 use scylla::Session as ScySession;
 use std::sync::Arc;
 
+#[derive(Debug, ThisError)]
+pub enum Error {
+    NewSessionError(#[from] NewSessionError),
+    QueryError(#[from] QueryError),
+}
+
 pub struct DataStore {
     pub scy: Arc<ScySession>,
     pub qu_insert_ts_msp: Arc<PreparedStatement>,
@@ -44,53 +52,44 @@ impl DataStore {
                     .into_handle(),
             )
             .build()
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let scy = Arc::new(scy);
 
         let q = scy
             .prepare("insert into ts_msp (series, ts_msp) values (?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_ts_msp = Arc::new(q);
         let q = scy
             .prepare(
                 "insert into series_by_ts_msp (part, ts_msp, shape_kind, scalar_type, series) values (?, ?, ?, ?, ?) using ttl ?",
             )
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await  ?;
         let qu_insert_series_by_ts_msp = Arc::new(q);
 
         // scalar:
         let q = scy
             .prepare("insert into events_scalar_i8 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_i8 = Arc::new(q);
         let q = scy
             .prepare("insert into events_scalar_i16 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_i16 = Arc::new(q);
         let q = scy
             .prepare("insert into events_scalar_i32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_i32 = Arc::new(q);
         let q = scy
             .prepare("insert into events_scalar_f32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_f32 = Arc::new(q);
         let q = scy
             .prepare("insert into events_scalar_f64 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_f64 = Arc::new(q);
         let q = scy
             .prepare("insert into events_scalar_string (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_scalar_string = Arc::new(q);
 
         // array
@@ -98,73 +97,60 @@ impl DataStore {
             .prepare(
                 "insert into events_array_i8 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?",
             )
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_i8 = Arc::new(q);
         let q = scy
             .prepare("insert into events_array_i16 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_i16 = Arc::new(q);
         let q = scy
             .prepare("insert into events_array_i32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_i32 = Arc::new(q);
         let q = scy
             .prepare("insert into events_array_f32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_f32 = Arc::new(q);
         let q = scy
             .prepare("insert into events_array_f64 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_f64 = Arc::new(q);
         let q = scy
             .prepare("insert into events_array_bool (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_array_bool = Arc::new(q);
         // Others:
         let q = scy
             .prepare("insert into muted (part, series, ts, ema, emd) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_muted = Arc::new(q);
         let q = scy
             .prepare("insert into item_recv_ivl (part, series, ts, ema, emd) values (?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_item_recv_ivl = Arc::new(q);
         // Connection status:
         let q = scy
             .prepare("insert into connection_status (ts_msp, ts_lsp, kind, addr) values (?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_connection_status = Arc::new(q);
         let q = scy
             .prepare("insert into channel_status (series, ts_msp, ts_lsp, kind) values (?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_channel_status = Arc::new(q);
         let q = scy
             .prepare(
                 "insert into channel_status_by_ts_msp (ts_msp, ts_lsp, series, kind) values (?, ?, ?, ?) using ttl ?",
             )
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_channel_status_by_ts_msp = Arc::new(q);
         let q = scy
             .prepare("insert into channel_ping (part, ts_msp, series, ivl, interest, evsize) values (?, ?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_channel_ping = Arc::new(q);
 
         let q = scy
             .prepare("insert into binned_scalar_f32_v01 (series, bin_len_sec, bin_count, off_msp, off_lsp, counts, mins, maxs, avgs) values (?, ?, ?, ?, ?, ?, ?, ?, ?) using ttl ?")
-            .await
-            .map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
+            .await?;
         let qu_insert_binned_scalar_f32_v01 = Arc::new(q);
         let ret = Self {
             scy,

netfetch/src/daemon_common.rs

@@ -43,7 +43,6 @@ impl DaemonEvent {
                 match &b.value {
                     None => format!("CaConnEvent/None"),
                     EchoTimeout => format!("CaConnEvent/EchoTimeout"),
-                    HealthCheckDone => format!("CaConnEvent/HealthCheckDone"),
                     ConnCommandResult(_) => format!("CaConnEvent/ConnCommandResult"),
                     EndOfStream => format!("CaConnEvent/EndOfStream"),
                 }

netfetch/src/insertworker.rs

@@ -126,7 +126,7 @@ pub async fn spawn_scylla_insert_workers(
     let mut jhs = Vec::new();
     let mut data_stores = Vec::new();
     for _ in 0..insert_scylla_sessions {
-        let data_store = Arc::new(DataStore::new(&scyconf).await?);
+        let data_store = Arc::new(DataStore::new(&scyconf).await.map_err(|e| Error::from(e.to_string()))?);
         data_stores.push(data_store);
     }
     for worker_ix in 0..insert_worker_count {

netfetch/src/zmtp.rs

@@ -10,6 +10,8 @@ use crate::zmtp::zmtpproto::SocketType;
 use crate::zmtp::zmtpproto::Zmtp;
 #[allow(unused)]
 use bytes::BufMut;
+use err::thiserror;
+use err::ThisError;
 use futures_util::Future;
 use futures_util::FutureExt;
 use futures_util::StreamExt;
@@ -22,7 +24,7 @@ use std::pin::Pin;
 use std::task::Context;
 use std::task::Poll;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, ThisError)]
 pub enum Error {
     #[error("Msg({0})")]
     Msg(String),

netfetch/src/zmtp/zmtpproto.rs

@@ -7,6 +7,7 @@ use crate::zmtp::ZmtpEvent;
 use async_channel::Receiver;
 use async_channel::Sender;
 use err::thiserror;
+use err::ThisError;
 use futures_util::pin_mut;
 use futures_util::Stream;
 use futures_util::StreamExt;
@@ -25,7 +26,7 @@ use tokio::io::AsyncWrite;
 use tokio::io::ReadBuf;
 use tokio::net::TcpStream;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, ThisError)]
 pub enum Error {
     #[error("bad")]
     Bad,
@@ -59,6 +60,7 @@ enum ConnState {
     ReadFrameShort,
     ReadFrameLong,
     ReadFrameBody(usize),
+    #[allow(unused)]
     LockScan(usize),
 }
 

stats/Cargo.toml

@@ -11,5 +11,5 @@ path = "src/stats.rs"
 stats_types = { path = "../stats_types" }
 stats_proc = { path = "../stats_proc" }
 log = { path = "../log" }
-err = { path = "../../daqbuffer/err" }
+err = { path = "../../daqbuffer/crates/err" }
 libc = "0.2"