daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update deps, error type

Browse Source
This commit is contained in:
Dominik Werder
2025-06-03 12:28:46 +02:00
parent 31389fa7e3
commit 5a64613727
27 changed files with 268 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
.cargo/cargo-lock
View File

@@ -115,12 +115,12 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
version = "3.0.7"
version = "3.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
dependencies = [
"anstyle",
"once_cell",
"once_cell_polyfill",
"windows-sys 0.59.0",
]
@@ -221,18 +221,18 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "autoerr"
version = "0.0.3"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "980f0fd954ab9bdb2440aac932951f01455dff9cae664bdaca4d98aeb75b1116"
checksum = "be8cd7d4dbe9d1f58cb80acbda5d1dccd35701f4ef5cfa37492b153e49dedc51"
dependencies = [
"autoerr-impl",
]
[[package]]
name = "autoerr-impl"
version = "0.0.3"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "745e965b2d8ddc2d25707087e42b550f67f9391366802185b0a3d8ac4a4511f7"
checksum = "7166e8bc0a13b079544c28a8144e006c560b52ff836b453240ae1b251b8cb57d"
dependencies = [
"proc-macro2",
"quote",
@@ -325,9 +325,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
[[package]]
name = "cc"
version = "1.2.23"
version = "1.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"
checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951"
dependencies = [
"shlex",
]
@@ -394,9 +394,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.38"
version = "4.5.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000"
checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f"
dependencies = [
"clap_builder",
"clap_derive",
@@ -404,9 +404,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.38"
version = "4.5.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120"
checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51"
dependencies = [
"anstream",
"anstyle",
@@ -484,9 +484,9 @@ dependencies = [
[[package]]
name = "core-foundation"
version = "0.10.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
@@ -604,10 +604,11 @@ dependencies = [
[[package]]
name = "daqbuf-err"
version = "0.0.6"
version = "0.0.7"
dependencies = [
"anyhow",
"async-channel 2.3.1",
"autoerr",
"backtrace",
"chrono",
"http",
@@ -616,7 +617,6 @@ dependencies = [
"serde",
"serde_cbor",
"serde_json",
"thiserror 0.0.1",
"url",
]
@@ -775,7 +775,6 @@ dependencies = [
"serde",
"serde_cbor",
"serde_json",
"thiserror 0.0.1",
"typetag",
]
@@ -811,8 +810,9 @@ dependencies = [
[[package]]
name = "daqretrieve"
version = "0.5.5-aa.13"
version = "0.5.5-aa.14"
dependencies = [
"autoerr",
"bytes",
"chrono",
"clap",
@@ -928,6 +928,7 @@ dependencies = [
"arrayref",
"async-channel 1.9.0",
"async-stream",
"autoerr",
"byteorder",
"bytes",
"chrono",
@@ -1078,6 +1079,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@@ -1320,6 +1327,11 @@ name = "hashbrown"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "heapless"
@@ -1516,12 +1528,13 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.11"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http",
"http-body",
@@ -1607,9 +1620,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
[[package]]
name = "icu_properties"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a"
checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
dependencies = [
"displaydoc",
"icu_collections",
@@ -1623,9 +1636,9 @@ dependencies = [
[[package]]
name = "icu_properties_data"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04"
checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
[[package]]
name = "icu_provider"
@@ -1772,9 +1785,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
[[package]]
name = "lock_api"
version = "0.4.12"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
@@ -1834,13 +1847,13 @@ dependencies = [
[[package]]
name = "mio"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -1954,6 +1967,12 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "openssl-probe"
version = "0.1.6"
@@ -1974,9 +1993,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
[[package]]
name = "parking_lot"
version = "0.12.3"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
dependencies = [
"lock_api",
"parking_lot_core",
@@ -1984,9 +2003,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.9.10"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
@@ -2481,9 +2500,9 @@ dependencies = [
[[package]]
name = "rustversion"
version = "1.0.20"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
[[package]]
name = "ryu"
@@ -2517,26 +2536,22 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scylla"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd81aa1c6de67ed43f78a64aa233c547c569979922ab2e6c6d58730f0d53b651"
checksum = "1f4d5b7fe3cb3e140d374238f5f916eb810053562287a01d66ac685e305c166f"
dependencies = [
"arc-swap",
"async-trait",
"byteorder",
"bytes",
"chrono",
"dashmap",
"futures",
"hashbrown 0.14.5",
"itertools 0.14.0",
"lazy_static",
"lz4_flex",
"rand 0.9.1",
"rand_pcg",
"scylla-cql",
"smallvec",
"snap",
"socket2",
"thiserror 2.0.12",
"tokio",
@@ -2546,11 +2561,10 @@ dependencies = [
[[package]]
name = "scylla-cql"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae84054fed47b0a7e757d66aeb4574a4ca9696bf68c25ed6325b0f198c15795b"
checksum = "507db4914c625c86d32c5c00ed1add75eaf966a2d4ba9772b601b2563701df58"
dependencies = [
"async-trait",
"byteorder",
"bytes",
"chrono",
@@ -2567,9 +2581,9 @@ dependencies = [
[[package]]
name = "scylla-macros"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e67d343ebea5cc079bfa89ca4b231b432207c3748baaa3b00e5c238e7644e838"
checksum = "8efb4b519ab70556c8d3adfd4f192c635d0c7c72d4f125d2b79713742c98f39d"
dependencies = [
"darling",
"proc-macro2",
@@ -2591,9 +2605,13 @@ dependencies = [
"daqbuf-series",
"daqbuf-streams",
"futures-util",
"hashbrown 0.15.3",
"pin-project",
"scylla",
"serde",
"serde_json",
"taskrun",
"time",
]
[[package]]
@@ -2750,9 +2768,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
name = "socket2"
version = "0.5.9"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
@@ -2784,7 +2802,6 @@ name = "streamio"
version = "0.0.2"
dependencies = [
"arrayref",
"async-channel 1.9.0",
"autoerr",
"byteorder",
"bytes",
@@ -2872,14 +2889,6 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "thiserror"
version = "0.0.1"
source = "git+https://github.com/dominikwerder/thiserror.git?branch=cstm#8d3fc303d3741068c05ce2b533c058fa44bf9a1d"
dependencies = [
"thiserror-impl 1.0.61",
]
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -2898,16 +2907,6 @@ dependencies = [
"thiserror-impl 2.0.12",
]
[[package]]
name = "thiserror-impl"
version = "1.0.61"
source = "git+https://github.com/dominikwerder/thiserror.git?branch=cstm#8d3fc303d3741068c05ce2b533c058fa44bf9a1d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
@@ -3007,9 +3006,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.45.0"
version = "1.45.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165"
checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
dependencies = [
"backtrace",
"bytes",
@@ -3278,11 +3277,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.16.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
dependencies = [
"getrandom 0.3.3",
"js-sys",
"wasm-bindgen",
]
[[package]]
@@ -3477,9 +3478,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.61.1"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46ec44dc15085cea82cf9c78f85a9114c463a369786585ad2882d1ff0b0acf40"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
"windows-implement",
"windows-interface",
@@ -3518,18 +3519,18 @@ checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
[[package]]
name = "windows-result"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b895b5356fc36103d0f64dd1e94dfa7ac5633f1c9dd6e80fe9ec4adef69e09d"
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a7ab927b2637c19b3dbe0965e75d8f2d30bdd697a1516191cad2ec4df8fb28a"
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
dependencies = [
"windows-link",
]

4
Cargo.toml
View File

@@ -20,7 +20,3 @@ debug-assertions = false
lto = "off"
codegen-units = 32
incremental = true
[patch.crates-io]
#tokio = { git = "https://github.com/dominikwerder/tokio", rev = "995221d8" }
thiserror = { git = "https://github.com/dominikwerder/thiserror.git", branch = "cstm" }

2
crates/daqbuf-redis/Cargo.toml
View File

@@ -7,6 +7,6 @@ edition = "2024"
[dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
autoerr = "0.0.3"
autoerr = "0.0"
taskrun = { path = "../taskrun" }
redis = { version = "0.31.0", features = [] }

3
crates/daqbuffer/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "daqretrieve"
version = "0.5.5-aa.13"
version = "0.5.5-aa.14"
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2024"
@@ -13,6 +13,7 @@ serde_json = "1.0"
serde_yaml = "0.9.34"
chrono = "0.4.39"
url = "2.5.4"
autoerr = "0.0.5"
clap = { version = "4.5.28", features = ["derive", "cargo"] }
daqbuf-err = { path = "../../../daqbuf-err" }
taskrun = { path = "../taskrun" }

33
crates/daqbuffer/src/fetch.rs
View File

@@ -1,32 +1,31 @@
use daqbuf_err::thiserror;
use daqbuf_err::ThisError;
use futures_util::future;
use futures_util::StreamExt;
use http::header;
use futures_util::future;
use http::Method;
use http::header;
use httpclient::IncomingStream;
use httpclient::body_empty;
use httpclient::connect_client;
use httpclient::http;
use httpclient::http::StatusCode;
use httpclient::hyper::Request;
use httpclient::IncomingStream;
use netpod::log::*;
use netpod::APP_CBOR_FRAMED;
use netpod::ScalarType;
use netpod::Shape;
use netpod::APP_CBOR_FRAMED;
use netpod::log::*;
use streams::cbor_stream::FramedBytesToChannelEventsStream;
use url::Url;
#[derive(Debug, ThisError)]
#[cstm(name = "DataFetch")]
pub enum Error {
Url(#[from] url::ParseError),
NoHostname,
HttpBody(#[from] http::Error),
HttpClient(#[from] httpclient::Error),
Hyper(#[from] httpclient::hyper::Error),
RequestFailed(String),
}
autoerr::create_error_v1!(
name(Error, "DataFetch"),
enum variants {
Url(#[from] url::ParseError),
NoHostname,
HttpBody(#[from] http::Error),
HttpClient(#[from] httpclient::Error),
Hyper(#[from] httpclient::hyper::Error),
RequestFailed(String),
},
);
pub async fn fetch_cbor(url: &str, scalar_type: ScalarType, shape: Shape) -> Result<(), Error> {
let url: Url = url.parse()?;

2
crates/daqbufp2/src/client.rs
View File

@@ -136,7 +136,7 @@ pub async fn get_binned(
"get_cached_0 DONE total download {} MB throughput {:5} kB/s bin_count {}",
ntot / 1024 / 1024,
throughput,
bin_count,
bin_count
);
Ok(())
}

2
crates/dbconn/Cargo.toml
View File

@@ -20,7 +20,7 @@ pin-project = "1"
async-channel = "1.9.0"
chrono = "0.4.39"
regex = "1.11.1"
autoerr = "0.0.3"
autoerr = "0.0.5"
daqbuf-err = { path = "../../../daqbuf-err" }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
parse = { path = "../../../daqbuf-parse", package = "daqbuf-parse" }

38
crates/dbconn/src/channelconfig.rs
View File

@@ -1,31 +1,29 @@
use autoerr::dbgdisplay::Dbg;
use chrono::DateTime;
use chrono::Utc;
use daqbuf_err as err;
use err::thiserror;
use err::ThisError;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::ChConf;
use netpod::ScalarType;
use netpod::SeriesKind;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsMs;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use std::time::Duration;
use tokio_postgres::Client;
#[derive(Debug, ThisError)]
#[cstm(name = "DbChannelConfig")]
pub enum Error {
Pg(#[from] tokio_postgres::Error),
#[error("NotFound({0}, {1})")]
NotFound(SfDbChannel, NanoRange),
SeriesNotFound(String, u64),
BadScalarType(i32),
BadShape(Vec<i32>),
BadKind(i16),
NoInput,
}
autoerr::create_error_v1!(
name(Error, "DbChannelConfig"),
enum variants {
Pg(#[from] tokio_postgres::Error),
NotFound(SfDbChannel, NanoRange),
SeriesNotFound(String, u64),
BadScalarType(i32),
BadShape(Dbg<Vec<i32>>),
BadKind(i16),
NoInput,
},
);
/// It is an unsolved question as to how we want to uniquely address channels.
/// Currently, the usual (backend, channelname) works in 99% of the cases, but the edge-cases
@@ -67,7 +65,7 @@ pub(super) async fn chconf_best_matching_for_name_and_range(
let series = series as u64;
let _scalar_type =
ScalarType::from_scylla_i32(scalar_type).map_err(|_| Error::BadScalarType(scalar_type))?;
let _shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadShape(shape_dims))?;
let _shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadShape(Dbg(shape_dims)))?;
let tsms = tsc.signed_duration_since(DateTime::UNIX_EPOCH).num_milliseconds() as u64;
let ts = TsMs::from_ms_u64(tsms);
rows.push((ts, series));
@@ -88,7 +86,7 @@ pub(super) async fn chconf_best_matching_for_name_and_range(
let series = series as u64;
let kind = channel.kind();
let scalar_type = ScalarType::from_scylla_i32(scalar_type).map_err(|_| Error::BadScalarType(scalar_type))?;
let shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadShape(shape_dims))?;
let shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadShape(Dbg(shape_dims)))?;
let ret = ChConf::new(channel.backend(), series, kind, scalar_type, shape, channel.name());
Ok(ret)
}
@@ -217,7 +215,7 @@ pub(super) async fn chconf_for_series(backend: &str, series: u64, pg: &Client) -
ScalarType::from_dtype_index(scalar_type as _).map_err(|_| Error::BadScalarType(scalar_type))?;
// TODO can I get a slice from psql driver?
let shape = row.get::<_, Vec<i32>>(2);
let shape = Shape::from_scylla_shape_dims(&shape).map_err(|_| Error::BadShape(shape))?;
let shape = Shape::from_scylla_shape_dims(&shape).map_err(|_| Error::BadShape(Dbg(shape)))?;
let kind: i16 = row.get(3);
let kind = SeriesKind::from_db_i16(kind).map_err(|_| Error::BadKind(kind))?;
let ret = ChConf::new(backend, series, kind, scalar_type, shape, name);

27
crates/dbconn/src/dbconn.rs
View File

@@ -5,24 +5,22 @@ pub mod search;
pub mod worker;
pub mod pg {
pub use tokio_postgres::types::Type;
pub use tokio_postgres::Client;
pub use tokio_postgres::Error;
pub use tokio_postgres::NoTls;
pub use tokio_postgres::Statement;
pub use tokio_postgres::types::Type;
}
use daqbuf_err as err;
use err::anyhow;
use err::thiserror;
use err::Error;
use err::Res2;
use err::ThisError;
use netpod::log::*;
use err::anyhow;
use netpod::Database;
use netpod::NodeConfigCached;
use netpod::SfDbChannel;
use netpod::TableSizes;
use netpod::log::*;
use pg::Client as PgClient;
use pg::NoTls;
use serde::Serialize;
@@ -195,12 +193,13 @@ pub async fn find_series_sf_databuffer(channel: &SfDbChannel, pgclient: Arc<PgCl
Ok(series)
}
#[derive(Debug, ThisError, Serialize)]
#[cstm(name = "FindChannel")]
pub enum FindChannelError {
UnknownBackend,
BadSeriesId,
NoFound,
MultipleFound,
Database(String),
}
autoerr::create_error_v1!(
name(FindChannelError, "FindChannel"),
enum variants {
UnknownBackend,
BadSeriesId,
NoFound,
MultipleFound,
Database(String),
},
);

1
crates/disk/Cargo.toml
View File

@@ -29,6 +29,7 @@ hex = "0.4.3"
num-traits = "0.2.19"
num-derive = "0.4.2"
url = "2.5.4"
autoerr = "0.0.5"
tiny-keccak = { version = "2.0", features = ["sha3"] }
daqbuf-err = { path = "../../../daqbuf-err" }
taskrun = { path = "../taskrun" }

15
crates/disk/src/channelconfig.rs
View File

@@ -21,13 +21,14 @@ use std::time::SystemTime;
use streams::tcprawclient::TEST_BACKEND;
use taskrun::tokio;
#[derive(Debug, ThisError)]
#[cstm(name = "ChannelConfig")]
pub enum ConfigError {
ParseError(ConfigParseError),
NotFound,
Error,
}
autoerr::create_error_v1!(
name(ConfigError, "ChannelConfig"),
enum variants {
ParseError(ConfigParseError),
NotFound,
Error,
},
);
impl From<ConfigParseError> for ConfigError {
fn from(value: ConfigParseError) -> Self {

5
crates/disk/src/dataopen.rs
View File

@@ -91,7 +91,10 @@ async fn position_file(
let gg = match gg {
Ok(x) => x,
Err(e) => {
error!("can not position file for range {range:?} expand_right {expand_right:?} buflen {buflen}", buflen = buf.len());
error!(
"can not position file for range {range:?} expand_right {expand_right:?} buflen {}",
buf.len()
);
return Err(e);
}
};

44
crates/disk/src/eventchunker.rs
View File

@@ -1,9 +1,7 @@
use bytes::Buf;
use bytes::BytesMut;
use daqbuf_err as err;
use err::thiserror;
use err::Error;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::LogItem;
@@ -23,8 +21,6 @@ use netpod::ScalarType;
use netpod::SfChFetchInfo;
use netpod::Shape;
use parse::channelconfig::CompressionMethod;
use serde::Deserialize;
use serde::Serialize;
use std::collections::VecDeque;
use std::io::Cursor;
use std::path::PathBuf;
@@ -38,26 +34,26 @@ use streams::needminbuffer::NeedMinBuffer;
#[allow(unused)]
macro_rules! trace_parse_buf { ($($arg:tt)*) => ( if false { trace!($($arg)*); }) }
#[derive(Debug, ThisError, Serialize, Deserialize)]
#[cstm(name = "DatabufferDataParse")]
pub enum DataParseError {
DataFrameLengthMismatch,
FileHeaderTooShort,
BadVersionTag,
HeaderTooLarge,
Utf8Error,
EventTooShort,
#[error("EventTooLong({0}, {1})")]
EventTooLong(Shape, u32),
TooManyBeforeRange,
EventWithOptional,
BadTypeIndex,
WaveShapeWithoutEventArray,
ShapedWithoutDims,
TooManyDims,
UnknownCompression,
BadCompresionBlockSize,
}
autoerr::create_error_v1!(
name(DataParseError, "DatabufferDataParse"),
enum variants {
DataFrameLengthMismatch,
FileHeaderTooShort,
BadVersionTag,
HeaderTooLarge,
Utf8Error,
EventTooShort,
EventTooLong(Shape, u32),
TooManyBeforeRange,
EventWithOptional,
BadTypeIndex,
WaveShapeWithoutEventArray,
ShapedWithoutDims,
TooManyDims,
UnknownCompression,
BadCompresionBlockSize,
},
);
pub struct EventChunker {
inp: NeedMinBuffer,

2
crates/httpclient/Cargo.toml
View File

@@ -19,7 +19,7 @@ hyper = { version = "1.6.0", features = ["http1", "http2", "client", "server"] }
hyper-util = { version = "0.1.10", features = ["full"] }
bytes = "1.10.0"
async-channel = "1.9.0"
autoerr = "0.0.3"
autoerr = "0.0"
daqbuf-err = { path = "../../../daqbuf-err" }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
parse = { path = "../../../daqbuf-parse", package = "daqbuf-parse" }

2
crates/httpret/Cargo.toml
View File

@@ -35,7 +35,7 @@ rand = "0.9.0"
ciborium = "0.2.2"
flate2 = "1"
brotli = "8.0.1"
autoerr = "0.0.3"
autoerr = "0.0"
daqbuf-err = { path = "../../../daqbuf-err" }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
query = { path = "../../../daqbuf-query", package = "daqbuf-query" }

2
crates/httpret/src/api1.rs
View File

@@ -623,7 +623,7 @@ impl DataApiPython3DataStream {
b.be[i1],
b.scalar_types[i1],
b.shapes[i1],
b.comps[i1],
b.comps[i1]
);
}
// TODO emit warning when we use a different setting compared to channel config.

25
crates/httpret/src/api4/databuffer_tools.rs
View File

@@ -3,9 +3,7 @@ use async_channel::Receiver;
use async_channel::Sender;
use bytes::Bytes;
use daqbuf_err as err;
use err::thiserror;
use err::PublicError;
use err::ThisError;
use err::ToPublicError;
use futures_util::Stream;
use futures_util::StreamExt;
@@ -30,18 +28,17 @@ use std::task::Context;
use std::task::Poll;
use taskrun::tokio;
#[derive(Debug, ThisError)]
#[cstm(name = "ChannelFindActive")]
pub enum FindActiveError {
HttpBadAccept,
HttpBadUrl,
#[error("Error({0})")]
Error(Box<dyn ToPublicError>),
#[error("UrlError({0})")]
UrlError(#[from] url::ParseError),
InternalError,
IO(#[from] std::io::Error),
}
autoerr::create_error_v1!(
name(FindActiveError, "ChannelFindActive"),
enum variants {
HttpBadAccept,
HttpBadUrl,
Error(Box<dyn ToPublicError>),
UrlError(#[from] url::ParseError),
InternalError,
IO(#[from] std::io::Error),
},
);
impl ToPublicError for FindActiveError {
fn to_public_error(&self) -> PublicError {

17
crates/httpret/src/api4/eventdata.rs
View File

@@ -2,9 +2,7 @@ use crate::response;
use crate::ReqCtx;
use crate::ServiceSharedResources;
use daqbuf_err as err;
use err::thiserror;
use err::PublicError;
use err::ThisError;
use err::ToPublicError;
use http::Method;
use http::StatusCode;
@@ -19,13 +17,14 @@ use netpod::NodeConfigCached;
use std::sync::Arc;
use streams::instrument::InstrumentStream;
#[derive(Debug, ThisError)]
#[cstm(name = "EventData")]
pub enum EventDataError {
QueryParse,
Error(Box<dyn ToPublicError>),
InternalError,
}
autoerr::create_error_v1!(
name(EventDataError, "EventData"),
enum variants {
QueryParse,
Error(Box<dyn ToPublicError>),
InternalError,
},
);
impl ToPublicError for EventDataError {
fn to_public_error(&self) -> PublicError {

24
crates/httpret/src/api4/events.rs
View File

@@ -4,10 +4,7 @@ use crate::requests::accepts_json_framed;
use crate::requests::accepts_json_or_all;
use crate::response;
use crate::ServiceSharedResources;
use daqbuf_err as err;
use dbconn::worker::PgQueue;
use err::thiserror;
use err::ThisError;
use http::header::CONTENT_TYPE;
use http::Method;
use http::StatusCode;
@@ -44,16 +41,17 @@ use streams::streamtimeout::StreamTimeout2;
use tracing::Instrument;
use tracing::Span;
#[derive(Debug, ThisError)]
#[cstm(name = "Api4Events")]
pub enum Error {
ChannelNotFound,
HttpLib(#[from] http::Error),
ChannelConfig(crate::channelconfig::Error),
Retrieval(#[from] crate::RetrievalError),
EventsCbor(#[from] streams::plaineventscbor::Error),
EventsJson(#[from] streams::plaineventsjson::Error),
}
autoerr::create_error_v1!(
name(Error, "Api4Events"),
enum variants {
ChannelNotFound,
HttpLib(#[from] http::Error),
ChannelConfig(crate::channelconfig::Error),
Retrieval(#[from] crate::RetrievalError),
EventsCbor(#[from] streams::plaineventscbor::Error),
EventsJson(#[from] streams::plaineventsjson::Error),
},
);
impl Error {
pub fn user_message(&self) -> String {

2
crates/nodenet/Cargo.toml
View File

@@ -17,7 +17,7 @@ byteorder = "1.5.0"
futures-util = "0.3.31"
tracing = "0.1.41"
hex = "0.4.3"
autoerr = "0.0.3"
autoerr = "0.0"
daqbuf-err = { path = "../../../daqbuf-err" }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
query = { path = "../../../daqbuf-query", package = "daqbuf-query" }

40
crates/nodenet/src/channelconfig.rs
View File

@@ -1,7 +1,4 @@
use daqbuf_err as err;
use dbconn::worker::PgQueue;
use err::thiserror;
use err::ThisError;
use httpclient::url::Url;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
@@ -22,24 +19,25 @@ use netpod::Shape;
use netpod::APP_JSON;
use serde::Serialize;
#[derive(Debug, ThisError)]
#[cstm(name = "ChannelConfigNode")]
pub enum Error {
NotFoundChannel(SfDbChannel),
ChannelConfig(dbconn::channelconfig::Error),
DbWorker(#[from] dbconn::worker::Error),
DiskConfig(#[from] disk::channelconfig::ConfigError),
BackendConfigError,
BadTestSetup,
HttpReqError,
HttpClient(#[from] httpclient::Error),
ConfigParse(#[from] disk::parse::channelconfig::ConfigParseError),
JsonParse(#[from] serde_json::Error),
SearchWithGivenSeries,
AsyncSend,
AsyncRecv,
Todo,
}
autoerr::create_error_v1!(
name(Error, "ChannelConfigNode"),
enum variants {
NotFoundChannel(SfDbChannel),
ChannelConfig(dbconn::channelconfig::Error),
DbWorker(#[from] dbconn::worker::Error),
DiskConfig(#[from] disk::channelconfig::ConfigError),
BackendConfigError,
BadTestSetup,
HttpReqError,
HttpClient(#[from] httpclient::Error),
ConfigParse(#[from] disk::parse::channelconfig::ConfigParseError),
JsonParse(#[from] serde_json::Error),
SearchWithGivenSeries,
AsyncSend,
AsyncRecv,
Todo,
},
);
impl From<async_channel::RecvError> for Error {
fn from(_value: async_channel::RecvError) -> Self {

30
crates/nodenet/src/configquorum.rs
View File

@@ -1,8 +1,5 @@
use crate::channelconfig::http_get_channel_config;
use daqbuf_err as err;
use dbconn::worker::PgQueue;
use err::thiserror;
use err::ThisError;
use netpod::log::*;
use netpod::range::evrange::SeriesRange;
use netpod::ChConf;
@@ -18,19 +15,20 @@ use std::collections::BTreeMap;
use std::time::Duration;
use taskrun::tokio;
#[derive(Debug, ThisError)]
#[cstm(name = "ConfigQuorum")]
pub enum Error {
NotFound(SfDbChannel),
MissingTimeRange,
Timeout,
ChannelConfig(crate::channelconfig::Error),
ExpectSfDatabufferBackend,
UnsupportedBackend,
BadTimeRange,
DbWorker(#[from] dbconn::worker::Error),
FindChannel(#[from] dbconn::FindChannelError),
}
autoerr::create_error_v1!(
name(Error, "ConfigQuorum"),
enum variants {
NotFound(SfDbChannel),
MissingTimeRange,
Timeout,
ChannelConfig(crate::channelconfig::Error),
ExpectSfDatabufferBackend,
UnsupportedBackend,
BadTimeRange,
DbWorker(#[from] dbconn::worker::Error),
FindChannel(#[from] dbconn::FindChannelError),
},
);
impl From<crate::channelconfig::Error> for Error {
fn from(value: crate::channelconfig::Error) -> Self {

37
crates/nodenet/src/conn.rs
View File

@@ -1,8 +1,6 @@
use crate::scylla::scylla_channel_event_stream;
use bytes::Bytes;
use daqbuf_err as err;
use err::thiserror;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
@@ -41,23 +39,24 @@ use tracing::Instrument;
#[cfg(test)]
mod test;
#[derive(Debug, ThisError)]
#[cstm(name = "NodenetConn")]
pub enum Error {
BadQuery,
Scylla(#[from] crate::scylla::Error),
Error(#[from] err::Error),
Io(#[from] std::io::Error),
Items(#[from] items_2::Error),
NotAvailable,
DebugTest,
Generator(#[from] streams::generators::Error),
Framable(#[from] items_2::framable::Error),
Frame(#[from] items_2::frame::Error),
InMem(#[from] streams::frames::inmem::Error),
FramedStream(#[from] streams::frames::Error),
Netpod(#[from] netpod::Error),
}
autoerr::create_error_v1!(
name(Error, "NodenetConn"),
enum variants {
BadQuery,
Scylla(#[from] crate::scylla::Error),
Error(#[from] err::Error),
Io(#[from] std::io::Error),
Items(#[from] items_2::Error),
NotAvailable,
DebugTest,
Generator(#[from] streams::generators::Error),
Framable(#[from] items_2::framable::Error),
Frame(#[from] items_2::frame::Error),
InMem(#[from] streams::frames::inmem::Error),
FramedStream(#[from] streams::frames::Error),
Netpod(#[from] netpod::Error),
},
);
pub async fn events_service(ncc: NodeConfigCached) -> Result<(), Error> {
let scyqueue = err::todoval();

2
crates/scyllaconn/Cargo.toml
View File

@@ -13,7 +13,7 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
time = { version = "0.3.41", features = ["parsing", "formatting", "macros"] }
hashbrown = "0.15.3"
autoerr = "0.0.3"
autoerr = "0.0"
daqbuf-err = { path = "../../../daqbuf-err" }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
query = { path = "../../../daqbuf-query", package = "daqbuf-query" }

33
crates/scyllaconn/src/events2/mergertchained.rs
View File

@@ -5,23 +5,21 @@ use crate::events2::onebeforeandbulk;
use crate::range::ScyllaSeriesRange;
use crate::worker::ScyllaQueue;
use daqbuf_err as err;
use err::thiserror;
use err::ThisError;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::WithLen;
use items_0::merge::DrainIntoNewResult;
use items_0::merge::MergeableTy;
use items_0::WithLen;
use items_2::channelevents::ChannelEvents;
use netpod::ChConf;
use netpod::TsNano;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::range::evrange::SeriesRange;
use netpod::stream_impl_tracer::StreamImplTracer;
use netpod::ttl::RetentionTime;
use netpod::ChConf;
use netpod::TsNano;
use std::collections::VecDeque;
use std::pin::Pin;
use std::task::Context;
@@ -51,17 +49,18 @@ macro_rules! tracer_loop_enter {
};
}
#[derive(Debug, ThisError)]
#[cstm(name = "EventsMergeRt")]
pub enum Error {
Input(#[from] crate::events2::onebeforeandbulk::Error),
Events(#[from] crate::events2::events::Error),
Logic,
OrderMin,
OrderMax,
LimitPoll,
LimitLoop,
}
autoerr::create_error_v1!(
name(Error, "EventsMergeRt"),
enum variants {
Input(#[from] crate::events2::onebeforeandbulk::Error),
Events(#[from] crate::events2::events::Error),
Logic,
OrderMin,
OrderMax,
LimitPoll,
LimitLoop,
},
);
#[allow(unused)]
enum Resolvable<F>
@@ -339,7 +338,7 @@ impl MergeRtsChained {
fn push_out_one_before(&mut self) {
if let Some(buf) = self.buf_before.take() {
trace_fetch!("push_out_one_before len {len:?}", len = buf.len());
trace_fetch!("push_out_one_before len {:?}", buf.len());
if buf.len() != 0 {
self.out.push_back(buf);
}

3
crates/streamio/Cargo.toml
View File

@@ -18,9 +18,8 @@ bytes = "1.10"
arrayref = "0.3.9"
crc32fast = "1.4.2"
byteorder = "1.5.0"
async-channel = "1.9.0"
rand_xoshiro = "0.7.0"
autoerr = "0.0.3"
autoerr = "0.0"
chrono = { version = "0.4.39", features = ["serde"] }
netpod = { path = "../../../daqbuf-netpod", package = "daqbuf-netpod" }
query = { path = "../../../daqbuf-query", package = "daqbuf-query" }

2
crates/streamio/src/tcprawclient.rs
View File

@@ -42,7 +42,7 @@ pub async fn x_processed_event_blobs_stream_from_node_tcp(
node: Node,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
let addr = format!("{}:{}", node.host, node.port_raw);
debug!("x_processed_event_blobs_stream_from_node to: {addr}",);
debug!("x_processed_event_blobs_stream_from_node to: {addr}");
let frame1 = make_node_command_frame(subq.clone())?;
let net = TcpStream::connect(addr.clone()).await?;
let (netin, mut netout) = net.into_split();