From 6cd821378b9f27fddd854211d726162881de7d31 Mon Sep 17 00:00:00 2001 From: Dominik Werder Date: Tue, 29 Apr 2025 15:16:38 +0200 Subject: [PATCH] Move to scylla 1.1 --- .cargo/cargo-lock | 431 +++++++++++++++++++++++---------- daqingest/src/tools.rs | 9 +- netfetch/src/metrics/delete.rs | 42 ++-- scywr/Cargo.toml | 3 +- scywr/src/access.rs | 5 +- scywr/src/futbatch.rs | 8 +- scywr/src/futbatchgen.rs | 8 +- scywr/src/futinsert.rs | 8 +- scywr/src/insertworker.rs | 49 +--- scywr/src/iteminsertqueue.rs | 24 +- scywr/src/schema.rs | 30 +-- scywr/src/session.rs | 10 +- scywr/src/store.rs | 10 +- scywr/src/tools.rs | 6 +- stats/Cargo.toml | 2 +- 15 files changed, 398 insertions(+), 247 deletions(-) diff --git a/.cargo/cargo-lock b/.cargo/cargo-lock index af140c1..1975c80 100644 --- a/.cargo/cargo-lock +++ b/.cargo/cargo-lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arc-swap" @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.87" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ "axum-core", "bytes", @@ -221,12 +221,12 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", + "futures-core", "http", "http-body", "http-body-util", @@ -314,9 +314,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.16" +version = "1.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" dependencies = [ "shlex", ] @@ -329,9 +329,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -366,14 +366,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 2.4.1", + "half 2.6.0", ] [[package]] name = "clap" -version = "4.5.32" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", "clap_derive", @@ -381,9 +381,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.32" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", @@ -495,6 +495,7 @@ name = "daqbuf-ca-proto" version = "0.0.4" dependencies = [ "async-channel", + "autoerr", "byteorder", "bytes", "chrono", @@ -503,10 +504,10 @@ dependencies = [ "futures-util", "humantime", "humantime-serde", + "mettrics", "serde", "serde_json", "slidebuf", - "thiserror 0.0.1", "tracing", ] @@ -644,6 +645,7 @@ dependencies = [ "autoerr", "chrono", "daqbuf-netpod", + "daqbuf-series", "humantime", "humantime-serde", "serde", @@ -663,12 +665,20 @@ dependencies = [ "daqbuf-series", "futures-util", "pin-project-lite", - "scylla", + "scylla 0.15.1", "serde", "smallvec", "taskrun", ] +[[package]] +name = "daqbuf-serde-helper" +version = "0.0.1" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "daqbuf-series" version = "0.0.2" @@ -703,7 +713,7 @@ dependencies = [ "humantime", "itertools 0.13.0", "pin-project", - "rand_xoshiro", + "rand_xoshiro 0.6.0", "serde", "serde_cbor", "serde_json", @@ -713,7 +723,7 @@ dependencies = [ [[package]] name = "daqingest" -version = "0.2.7-aa.7" +version = "0.3.0-aa.2" dependencies = [ "async-channel", "autoerr", @@ -741,9 +751,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -751,9 +761,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", @@ -765,9 +775,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", @@ -824,11 +834,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -900,9 +911,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ "event-listener", "pin-project-lite", @@ -922,9 +933,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" @@ -1036,9 +1047,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", @@ -1047,14 +1058,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1071,9 +1082,9 @@ checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "half" -version = "2.4.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -1233,9 +1244,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-util", @@ -1249,14 +1260,15 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log 0.4.27", "wasm-bindgen", "windows-core", ] @@ -1311,9 +1323,9 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" [[package]] name = "icu_normalizer" @@ -1335,9 +1347,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" @@ -1356,9 +1368,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" @@ -1417,9 +1429,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1429,10 +1441,10 @@ dependencies = [ name = "ingest-linux" version = "0.0.1" dependencies = [ + "autoerr", "libc", "log 0.0.1", "taskrun", - "thiserror 0.0.1", ] [[package]] @@ -1492,9 +1504,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "litemap" @@ -1521,9 +1533,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lz4_flex" @@ -1556,6 +1568,27 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mettrics" +version = "0.0.7" +dependencies = [ + "mettrics-macros", + "serde", + "serde_json", +] + +[[package]] +name = "mettrics-macros" +version = "0.0.7" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + [[package]] name = "mime" version = "0.3.17" @@ -1570,9 +1603,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -1615,6 +1648,7 @@ dependencies = [ "daqbuf-items-0", "daqbuf-items-2", "daqbuf-netpod", + "daqbuf-serde-helper", "daqbuf-series", "daqbuf-streams", "dashmap 6.1.0", @@ -1636,7 +1670,6 @@ dependencies = [ "scywr", "serde", "serde_cbor", - "serde_helper", "serde_json", "serde_yaml", "serieswriter", @@ -1644,6 +1677,7 @@ dependencies = [ "smallvec", "stats", "taskrun", + "time", "tokio-stream", "tracing", "url", @@ -1695,9 +1729,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "overload" @@ -1822,7 +1856,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.0", + "rand 0.9.1", "sha2", "stringprep", ] @@ -1853,14 +1887,24 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.23", + "zerocopy 0.8.25", +] + +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -1874,6 +1918,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -1887,13 +1937,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.23", ] [[package]] @@ -1922,7 +1971,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -1931,7 +1980,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.2", ] [[package]] @@ -1943,6 +1992,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_pcg" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48ac3f7ffaab7fac4d2376632268aa5f89abdb55f7ebf8f4d11fffccb2320f7" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -1953,10 +2011,19 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.5.10" +name = "rand_xoshiro" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "redox_syscall" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags", ] @@ -2064,9 +2131,38 @@ dependencies = [ "lazy_static", "lz4_flex", "rand 0.8.5", - "rand_pcg", - "scylla-cql", - "scylla-macros", + "rand_pcg 0.3.1", + "scylla-cql 0.4.1", + "scylla-macros 0.7.1", + "smallvec", + "snap", + "socket2", + "thiserror 2.0.12", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "scylla" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd81aa1c6de67ed43f78a64aa233c547c569979922ab2e6c6d58730f0d53b651" +dependencies = [ + "arc-swap", + "async-trait", + "byteorder", + "bytes", + "chrono", + "dashmap 5.5.3", + "futures", + "hashbrown 0.14.5", + "itertools 0.14.0", + "lazy_static", + "lz4_flex", + "rand 0.9.1", + "rand_pcg 0.9.0", + "scylla-cql 1.1.0", "smallvec", "snap", "socket2", @@ -2086,7 +2182,28 @@ dependencies = [ "byteorder", "bytes", "lz4_flex", - "scylla-macros", + "scylla-macros 0.7.1", + "snap", + "stable_deref_trait", + "thiserror 2.0.12", + "tokio", + "uuid", + "yoke", +] + +[[package]] +name = "scylla-cql" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae84054fed47b0a7e757d66aeb4574a4ca9696bf68c25ed6325b0f198c15795b" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "chrono", + "itertools 0.14.0", + "lz4_flex", + "scylla-macros 1.1.0", "snap", "stable_deref_trait", "thiserror 2.0.12", @@ -2107,6 +2224,18 @@ dependencies = [ "syn", ] +[[package]] +name = "scylla-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67d343ebea5cc079bfa89ca4b231b432207c3748baaa3b00e5c238e7644e838" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "scywr" version = "0.0.1" @@ -2120,7 +2249,7 @@ dependencies = [ "futures-util", "log 0.0.1", "pin-project", - "scylla", + "scylla 1.1.0", "serde", "smallvec", "stats", @@ -2163,13 +2292,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_helper" -version = "0.0.1" -dependencies = [ - "serde", -] - [[package]] name = "serde_json" version = "1.0.140" @@ -2266,9 +2388,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -2299,9 +2421,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "snap" @@ -2311,9 +2433,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2344,8 +2466,11 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" name = "stats" version = "0.0.1-a.0" dependencies = [ + "daqbuf-ca-proto", "log 0.0.1", - "rand_xoshiro", + "mettrics", + "rand_xoshiro 0.7.0", + "serde", "stats_proc", "stats_types", ] @@ -2391,9 +2516,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -2482,9 +2607,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.39" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -2497,15 +2622,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -2538,9 +2663,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -2578,14 +2703,14 @@ dependencies = [ "fallible-iterator", "futures-channel", "futures-util", - "log 0.4.26", + "log 0.4.27", "parking_lot", "percent-encoding", "phf", "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.1", "socket2", "tokio", "tokio-util", @@ -2605,9 +2730,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -2650,7 +2775,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ - "log 0.4.26", + "log 0.4.27", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2683,7 +2808,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "log 0.4.26", + "log 0.4.27", "once_cell", "tracing-core", ] @@ -2813,11 +2938,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.2", ] [[package]] @@ -2840,9 +2965,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -2872,7 +2997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", - "log 0.4.26", + "log 0.4.27", "proc-macro2", "quote", "syn", @@ -2923,9 +3048,9 @@ dependencies = [ [[package]] name = "whoami" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ "redox_syscall", "wasite", @@ -2956,18 +3081,62 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-targets", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] [[package]] name = "windows-sys" @@ -3053,9 +3222,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags", ] @@ -3107,11 +3276,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.23" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "zerocopy-derive 0.8.23", + "zerocopy-derive 0.8.25", ] [[package]] @@ -3127,9 +3296,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.23" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", diff --git a/daqingest/src/tools.rs b/daqingest/src/tools.rs index 9a04620..3385a83 100644 --- a/daqingest/src/tools.rs +++ b/daqingest/src/tools.rs @@ -17,9 +17,7 @@ use netpod::Shape; use netpod::TsMs; use netpod::ttl::RetentionTime; use scywr::config::ScyllaIngestConfig; -use scywr::scylla::prepared_statement::PreparedStatement; -use scywr::scylla::transport::errors::QueryError; -use scywr::scylla::transport::iterator::NextRowError; +use scywr::scylla::statement::prepared::PreparedStatement; use scywr::session::ScySession; use series::SeriesId; use std::sync::Arc; @@ -32,8 +30,9 @@ autoerr::create_error_v1!( PgConn(#[from] dbpg::err::Error), Postgres(#[from] dbpg::postgres::Error), ScyllaSession(#[from] scywr::session::Error), - ScyllaQuery(#[from] QueryError), - ScyllaNextRowError(#[from] NextRowError), + ScyllaPrepareError(#[from] scywr::scylla::errors::PrepareError), + ScyllaPagerExecutionError(#[from] scywr::scylla::errors::PagerExecutionError), + ScyllaNextRowError(#[from] scywr::scylla::errors::NextRowError), ScyllaSchema(#[from] scywr::schema::Error), ScyllaTypeCheck(#[from] scywr::scylla::deserialize::TypeCheckError), ParseError(String), diff --git a/netfetch/src/metrics/delete.rs b/netfetch/src/metrics/delete.rs index 147a3f3..53fb8a4 100644 --- a/netfetch/src/metrics/delete.rs +++ b/netfetch/src/metrics/delete.rs @@ -1,21 +1,21 @@ use super::RoutesResources; +use axum::Json; use axum::extract::FromRequest; use axum::extract::Query; use axum::handler::Handler; use axum::http::HeaderMap; -use axum::Json; use bytes::Bytes; use chrono::DateTime; use chrono::Utc; use core::fmt; use futures_util::StreamExt; use futures_util::TryStreamExt; -use netpod::log::*; -use netpod::ttl::RetentionTime; use netpod::ScalarType; use netpod::TsMs; use netpod::TsNano; -use scylla::Session as ScySession; +use netpod::log::*; +use netpod::ttl::RetentionTime; +use scylla::client::session::Session as ScySession; use scywr::config::ScyllaIngestConfig; use scywr::insertqueues::InsertDeques; use scywr::iteminsertqueue::ArrayValue; @@ -23,7 +23,12 @@ use scywr::iteminsertqueue::DataValue; use scywr::iteminsertqueue::QueryItem; use scywr::iteminsertqueue::ScalarValue; use scywr::scylla; -use scywr::scylla::prepared_statement::PreparedStatement; +use scywr::scylla::client::PoolSize; +use scywr::scylla::client::execution_profile::ExecutionProfileBuilder; +use scywr::scylla::client::session_builder::GenericSessionBuilder; +use scywr::scylla::statement::Consistency; +use scywr::scylla::statement::Statement; +use scywr::scylla::statement::prepared::PreparedStatement; use serde::Deserialize; use series::SeriesId; use serieswriter::writer::SeriesWriter; @@ -37,13 +42,7 @@ use streams::framed_bytes::FramedBytesStream; use taskrun::tokio::time::timeout; #[allow(unused)] -macro_rules! debug_cql { - ($($arg:tt)*) => { - if true { - debug!($($arg)*); - } - }; -} +macro_rules! debug_cql { ($($arg:expr),*) => ( if false { debug!($($arg),*); } ); } autoerr::create_error_v1!( name(Error, "HttpDelete"), @@ -54,9 +53,10 @@ autoerr::create_error_v1!( MissingScalarType, MissingBegDate, MissingEndDate, - ScyllaTransport(#[from] scylla::transport::errors::NewSessionError), - ScyllaQuery(#[from] scylla::transport::errors::QueryError), - ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError), + ScyllaTransport(#[from] scylla::errors::NewSessionError), + ScyllaPrepare(#[from] scylla::errors::PrepareError), + ScyllaPagerExecution(#[from] scylla::errors::PagerExecutionError), + ScyllaNextRow(#[from] scylla::errors::NextRowError), ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError), InvalidTimestamp, }, @@ -126,7 +126,7 @@ async fn delete_try( scyconf.keyspace(), rt.table_prefix(), ); - scy.prepare(scylla::query::Query::new(cql).with_page_size(4)).await? + scy.prepare(Statement::new(cql).with_page_size(4)).await? }; let qu_delete_val = { let _cql = format!( @@ -149,10 +149,10 @@ async fn delete_try( rt.table_prefix(), scalar_type.to_scylla_table_name_id(), ); - scy.prepare(scylla::query::Query::new(cql).with_page_size(100)).await? + scy.prepare(Statement::new(cql).with_page_size(100)).await? }; let mut i = 0; - // debug_cql!("query iteration {i}"); + debug_cql!("query iteration {i}"); let mut it = scy .execute_iter(qu.clone(), (series.to_i64(),)) .await? @@ -176,7 +176,7 @@ async fn delete_val( ) -> Result<(), Error> { let msp_ns = msp.ns_u64(); if msp_ns >= end.ns() { - // debug_cql!(" return early msp {msp} after range"); + debug_cql!(" return early msp {msp} after range"); return Ok(()); } let r1 = if msp_ns >= beg.ns() { 0 } else { beg.ns() - msp_ns }; @@ -195,10 +195,6 @@ async fn delete_val( } async fn scy_connect(scyconf: &ScyllaIngestConfig) -> Result, Error> { - use scylla::execution_profile::ExecutionProfileBuilder; - use scylla::statement::Consistency; - use scylla::transport::session::PoolSize; - use scylla::transport::session_builder::GenericSessionBuilder; let profile = ExecutionProfileBuilder::default() .consistency(Consistency::Quorum) .build() diff --git a/scywr/Cargo.toml b/scywr/Cargo.toml index 2da1185..8aac481 100644 --- a/scywr/Cargo.toml +++ b/scywr/Cargo.toml @@ -7,7 +7,8 @@ edition = "2024" [dependencies] futures-util = "0.3.28" async-channel = "2.3.1" -scylla = "0.15.1" +#scylla = "0.15.1" +scylla = "1.1" smallvec = "1.11.0" pin-project = "1.1.5" bytes = "1.10.0" diff --git a/scywr/src/access.rs b/scywr/src/access.rs index 964a59b..798baba 100644 --- a/scywr/src/access.rs +++ b/scywr/src/access.rs @@ -1,11 +1,10 @@ -use scylla::transport::errors::DbError; -use scylla::transport::errors::QueryError; +use scylla::errors::DbError; autoerr::create_error_v1!( name(Error, "ScyllaAccess"), enum variants { DbError(#[from] DbError), - QueryError(#[from] QueryError), + Execution(#[from] scylla::errors::ExecutionError), NoKeyspaceChosen, }, ); diff --git a/scywr/src/futbatch.rs b/scywr/src/futbatch.rs index f6d5b43..106f4df 100644 --- a/scywr/src/futbatch.rs +++ b/scywr/src/futbatch.rs @@ -3,17 +3,17 @@ use crate::session::ScySession; use futures_util::Future; use futures_util::FutureExt; use netpod::log::*; -use scylla::batch::Batch; +use scylla::errors::ExecutionError; +use scylla::response::query_result::QueryResult; use scylla::serialize::batch::BatchValues; -use scylla::transport::errors::QueryError; -use scylla::QueryResult; +use scylla::statement::batch::Batch; use std::pin::Pin; use std::task::Context; use std::task::Poll; use std::time::Instant; pub struct ScyBatchFut<'a> { - fut: Pin> + 'a>>, + fut: Pin> + 'a>>, polled: usize, ts_create: Instant, ts_poll_start: Instant, diff --git a/scywr/src/futbatchgen.rs b/scywr/src/futbatchgen.rs index 2de6240..d9626fe 100644 --- a/scywr/src/futbatchgen.rs +++ b/scywr/src/futbatchgen.rs @@ -3,17 +3,17 @@ use crate::session::ScySession; use futures_util::Future; use futures_util::FutureExt; use netpod::log::*; -use scylla::batch::Batch; +use scylla::errors::ExecutionError; +use scylla::response::query_result::QueryResult; use scylla::serialize::batch::BatchValues; -use scylla::transport::errors::QueryError; -use scylla::QueryResult; +use scylla::statement::batch::Batch; use std::pin::Pin; use std::task::Context; use std::task::Poll; use std::time::Instant; pub struct ScyBatchFutGen<'a> { - fut: Pin> + Send + 'a>>, + fut: Pin> + Send + 'a>>, polled: usize, ts_create: Instant, ts_poll_start: Instant, diff --git a/scywr/src/futinsert.rs b/scywr/src/futinsert.rs index 6e4a137..9857049 100644 --- a/scywr/src/futinsert.rs +++ b/scywr/src/futinsert.rs @@ -3,17 +3,17 @@ use crate::session::ScySession; use futures_util::Future; use futures_util::FutureExt; use netpod::log::error; -use scylla::QueryResult; -use scylla::prepared_statement::PreparedStatement; +use scylla::errors::ExecutionError; +use scylla::response::query_result::QueryResult; use scylla::serialize::row::SerializeRow; -use scylla::transport::errors::QueryError; +use scylla::statement::prepared::PreparedStatement; use std::pin::Pin; use std::task::Context; use std::task::Poll; use std::time::Instant; pub struct ScyInsertFut<'a> { - fut: Pin> + Send + 'a>>, + fut: Pin> + Send + 'a>>, polled: usize, ts_create: Instant, ts_poll_first: Instant, diff --git a/scywr/src/insertworker.rs b/scywr/src/insertworker.rs index 0e8a0fd..8fdc7f1 100644 --- a/scywr/src/insertworker.rs +++ b/scywr/src/insertworker.rs @@ -53,34 +53,21 @@ autoerr::create_error_v1!( }, ); -fn stats_inc_for_err(stats: &stats::InsertWorkerStats, err: &crate::iteminsertqueue::Error) { - use crate::iteminsertqueue::Error; +fn stats_inc_for_err(stats: &stats::InsertWorkerStats, err: &crate::iteminsertqueue::InsertFutError) { + use crate::iteminsertqueue::InsertFutError; match err { - Error::DbOverload => { - stats.db_overload().inc(); - } - Error::DbTimeout => { - stats.db_timeout().inc(); - } - Error::DbUnavailable => { - stats.db_unavailable().inc(); - } - Error::DbError(_) => { - if true { - warn!("db error {}", err); + InsertFutError::Execution(e) => match e { + scylla::errors::ExecutionError::RequestTimeout(_) => { + stats.db_timeout().inc(); } - stats.db_error().inc(); - } - Error::QueryError(_) => { - stats.query_error().inc(); - } - Error::GetValHelpTodoWaveform => { - stats.logic_error().inc(); - } - Error::GetValHelpInnerTypeMismatch => { - stats.logic_error().inc(); - } - Error::UnknownConnectionStatus => { + _ => { + if true { + warn!("db error {}", err); + } + stats.db_error().inc(); + } + }, + InsertFutError::NoFuture => { stats.logic_error().inc(); } } @@ -266,16 +253,6 @@ async fn worker_streamed( mett.job_dt_net().push_dur_100us(dt_net); } Err(e) => { - use scylla::transport::errors::QueryError; - let e = match e { - QueryError::TimeoutError => crate::iteminsertqueue::Error::DbTimeout, - // TODO use `msg` - QueryError::DbError(e, _msg) => match e { - scylla::transport::errors::DbError::Overloaded => crate::iteminsertqueue::Error::DbOverload, - _ => e.into(), - }, - _ => e.into(), - }; mett.job_err().inc(); stats_inc_for_err(&stats, &e); } diff --git a/scywr/src/iteminsertqueue.rs b/scywr/src/iteminsertqueue.rs index 482486d..dea5178 100644 --- a/scywr/src/iteminsertqueue.rs +++ b/scywr/src/iteminsertqueue.rs @@ -5,18 +5,18 @@ use crate::store::DataStore; use bytes::BufMut; use futures_util::Future; use futures_util::FutureExt; +use futures_util::TryFutureExt; use netpod::DtNano; use netpod::Shape; use netpod::TsMs; use netpod::TsNano; use netpod::channelstatus::ChannelStatus; use netpod::channelstatus::ChannelStatusClosedReason; -use scylla::QueryResult; -use scylla::prepared_statement::PreparedStatement; +use scylla::errors::DbError; +use scylla::response::query_result::QueryResult; use scylla::serialize::row::SerializeRow; use scylla::serialize::value::SerializeValue; -use scylla::transport::errors::DbError; -use scylla::transport::errors::QueryError; +use scylla::statement::prepared::PreparedStatement; use series::ChannelStatusSeriesId; use series::SeriesId; use std::net::SocketAddrV4; @@ -35,13 +35,20 @@ autoerr::create_error_v1!( DbOverload, DbUnavailable, DbError(#[from] DbError), - QueryError(#[from] QueryError), GetValHelpTodoWaveform, GetValHelpInnerTypeMismatch, UnknownConnectionStatus, }, ); +autoerr::create_error_v1!( + name(InsertFutError, "InsertFut"), + enum variants { + NoFuture, + Execution(#[from] scylla::errors::ExecutionError), + }, +); + #[derive(Clone, Debug, PartialEq)] pub enum ScalarValue { U8(u8), @@ -633,7 +640,7 @@ pub struct InsertFut { scy: Arc, #[allow(unused)] qu: Arc, - fut: Pin> + Send>>, + fut: Pin> + Send>>, // #[pin] // fut: StackFuture<'static, Result, { 1024 * 3 }>, } @@ -644,6 +651,7 @@ impl InsertFut { let qu_ref = unsafe { NonNull::from(qu.as_ref()).as_ref() }; let fut = scy_ref.execute_unpaged(qu_ref, params); let fut = taskrun::tokio::task::unconstrained(fut); + let fut = fut.map_err(|e| e.into()); let fut = Box::pin(fut); // let _ff = StackFuture::from(fut); Self { scy, qu, fut } @@ -653,13 +661,13 @@ impl InsertFut { Self { scy, qu, - fut: Box::pin(async { Err(QueryError::TimeoutError) }), + fut: Box::pin(async { Err(InsertFutError::NoFuture) }), } } } impl Future for InsertFut { - type Output = Result; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { let this = self.project(); diff --git a/scywr/src/schema.rs b/scywr/src/schema.rs index fc3d280..984873a 100644 --- a/scywr/src/schema.rs +++ b/scywr/src/schema.rs @@ -5,7 +5,7 @@ use futures_util::StreamExt; use futures_util::TryStreamExt; use log::*; use netpod::ttl::RetentionTime; -use scylla::transport::errors::DbError; +use scylla::errors::NextRowError; use std::collections::BTreeMap; use std::fmt; use std::time::Duration; @@ -15,9 +15,10 @@ autoerr::create_error_v1!( enum variants { NoKeyspaceChosen, Fmt(#[from] fmt::Error), - Query(#[from] scylla::transport::errors::QueryError), NewSession(String), - ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError), + ScyllaExecution(#[from] scylla::errors::ExecutionError), + ScyllaPagerExecution(#[from] scylla::errors::PagerExecutionError), + ScyllaNextRow(#[from] NextRowError), ScyllaTypecheck(#[from] scylla::deserialize::TypeCheckError), MissingData, AddColumnExists(String, String, String), @@ -82,24 +83,23 @@ pub async fn has_table(ks: &str, name: &str, scy: &ScySession) -> Result Result { - use crate::scylla::transport::errors::QueryError; + use scylla::errors::ExecutionError; match scy .query_unpaged(format!("select * from {}.{} limit 1", ks, name), ()) .await { Ok(_) => Ok(true), Err(e) => match &e { - QueryError::DbError(e2, msg) => match e2 { - DbError::Invalid => { - if msg.contains("unconfigured table") { - Ok(false) - } else { - Err(e.into()) - } - } - _ => Err(e.into()), - }, - _ => Err(e.into()), + ExecutionError::BadQuery(_) => Ok(false), + ExecutionError::PrepareError(_) => Ok(false), + ExecutionError::EmptyPlan => Err(Error::ScyllaExecution(e)), + ExecutionError::ConnectionPoolError(_) => Err(Error::ScyllaExecution(e)), + ExecutionError::LastAttemptError(_) => Err(Error::ScyllaExecution(e)), + ExecutionError::RequestTimeout(_) => Err(Error::ScyllaExecution(e)), + ExecutionError::UseKeyspaceError(_) => Err(Error::ScyllaExecution(e)), + ExecutionError::SchemaAgreementError(_) => Err(Error::ScyllaExecution(e)), + ExecutionError::MetadataError(_) => Err(Error::ScyllaExecution(e)), + _ => Err(Error::ScyllaExecution(e)), }, } } diff --git a/scywr/src/session.rs b/scywr/src/session.rs index 911da0d..e366b4f 100644 --- a/scywr/src/session.rs +++ b/scywr/src/session.rs @@ -1,10 +1,12 @@ -pub use scylla::Session; pub use Session as ScySession; +pub use scylla::client::session::Session; use crate::config::ScyllaIngestConfig; -use scylla::execution_profile::ExecutionProfileBuilder; +use scylla::client::PoolSize; +use scylla::client::execution_profile::ExecutionProfileBuilder; +use scylla::client::session_builder::GenericSessionBuilder; +use scylla::errors::NewSessionError; use scylla::statement::Consistency; -use scylla::transport::errors::NewSessionError; use std::sync::Arc; autoerr::create_error_v1!( @@ -21,8 +23,6 @@ impl From for Error { } pub async fn create_session_no_ks(scyconf: &ScyllaIngestConfig) -> Result, Error> { - use scylla::transport::session::PoolSize; - use scylla::transport::session_builder::GenericSessionBuilder; let profile = ExecutionProfileBuilder::default() .consistency(Consistency::LocalOne) .build() diff --git a/scywr/src/store.rs b/scywr/src/store.rs index dc0ad8b..fa04159 100644 --- a/scywr/src/store.rs +++ b/scywr/src/store.rs @@ -1,17 +1,17 @@ use crate::config::ScyllaIngestConfig; use crate::session::create_session; use netpod::ttl::RetentionTime; -use scylla::Session as ScySession; -use scylla::prepared_statement::PreparedStatement; -use scylla::transport::errors::NewSessionError; -use scylla::transport::errors::QueryError; +use scylla::client::session::Session as ScySession; +use scylla::errors::NewSessionError; +use scylla::errors::PrepareError; +use scylla::statement::prepared::PreparedStatement; use std::sync::Arc; autoerr::create_error_v1!( name(Error, "ScyllaStore"), enum variants { NewSessionError(#[from] NewSessionError), - QueryError(#[from] QueryError), + Prepare(#[from] PrepareError), NewSession, }, ); diff --git a/scywr/src/tools.rs b/scywr/src/tools.rs index 719a634..7d96031 100644 --- a/scywr/src/tools.rs +++ b/scywr/src/tools.rs @@ -7,8 +7,10 @@ autoerr::create_error_v1!( name(Error, "ScyllaTools"), enum variants { Session(#[from] crate::session::Error), - ScyllaNewSession(#[from] scylla::transport::errors::NewSessionError), - ScyllaQueryError(#[from] scylla::transport::errors::QueryError), + ScyllaNewSession(#[from] scylla::errors::NewSessionError), + ScyllaPrepare(#[from] scylla::errors::PrepareError), + ScyllaNextRow(#[from] scylla::errors::NextRowError), + ScyllaPagerExecution(#[from] scylla::errors::PagerExecutionError), ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError), }, ); diff --git a/stats/Cargo.toml b/stats/Cargo.toml index 46f1ce9..642e577 100644 --- a/stats/Cargo.toml +++ b/stats/Cargo.toml @@ -9,7 +9,7 @@ path = "src/stats.rs" [dependencies] serde = { version = "1", features = ["derive"] } -rand_xoshiro = "0.6.0" +rand_xoshiro = "0.7.0" stats_types = { path = "../stats_types" } stats_proc = { path = "../stats_proc" } log = { path = "../log" }