diff --git a/.cargo/cargo-lock b/.cargo/cargo-lock index 9951c84..b9fbfec 100644 --- a/.cargo/cargo-lock +++ b/.cargo/cargo-lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -61,9 +61,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -76,43 +76,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arc-swap" @@ -126,17 +126,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - [[package]] name = "async-channel" version = "2.3.1" @@ -169,12 +158,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.4.0" @@ -183,9 +166,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.6" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", @@ -207,7 +190,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tower", "tower-layer", @@ -217,9 +200,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -230,7 +213,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -259,10 +242,10 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "batchtools" -version = "0.0.1" +version = "0.0.2" dependencies = [ - "async-channel 2.3.1", - "err", + "async-channel", + "daqbuf-err", "log 0.0.1", "taskrun", ] @@ -282,14 +265,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "bitshuffle" -version = "0.0.2" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -313,15 +288,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.1.22" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "shlex", ] @@ -376,9 +351,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b" dependencies = [ "clap_builder", "clap_derive", @@ -386,9 +361,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1" dependencies = [ "anstream", "anstyle", @@ -410,9 +385,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "cobs" @@ -422,9 +397,9 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "concurrent-queue" @@ -443,9 +418,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -461,9 +436,9 @@ dependencies = [ [[package]] name = "critical-section" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-utils" @@ -488,24 +463,218 @@ dependencies = [ ] [[package]] -name = "daqingest" -version = "0.2.4-aa.1" +name = "daqbuf-bitshuffle" +version = "0.0.3" dependencies = [ - "async-channel 2.3.1", + "cc", + "libc", +] + +[[package]] +name = "daqbuf-ca-proto" +version = "0.0.4" +dependencies = [ + "async-channel", + "byteorder", + "bytes", + "chrono", + "daqbuf-netpod", + "daqbuf-series", + "futures-util", + "humantime", + "humantime-serde", + "serde", + "serde_json", + "slidebuf", + "thiserror 0.0.1", + "tracing", +] + +[[package]] +name = "daqbuf-ca-proto-tokio" +version = "0.1.0" +dependencies = [ + "futures-util", + "pin-project", + "tokio", +] + +[[package]] +name = "daqbuf-err" +version = "0.0.6" +dependencies = [ + "anyhow", + "async-channel", + "backtrace", + "chrono", + "http", + "regex", + "rmp-serde", + "serde", + "serde_cbor", + "serde_json", + "thiserror 0.0.1", + "url", +] + +[[package]] +name = "daqbuf-items-0" +version = "0.0.3" +dependencies = [ + "bincode", + "bytes", + "chrono", + "ciborium", + "daqbuf-err", + "daqbuf-netpod", + "erased-serde", + "futures-util", + "serde", + "serde_json", + "thiserror 0.0.1", + "typetag", +] + +[[package]] +name = "daqbuf-items-2" +version = "0.0.4" +dependencies = [ + "bytes", + "chrono", + "ciborium", + "crc32fast", + "daqbuf-bitshuffle", + "daqbuf-err", + "daqbuf-items-0", + "daqbuf-items-proc", + "daqbuf-netpod", + "daqbuf-parse", + "erased-serde", + "futures-util", + "humantime-serde", + "num-traits", + "postcard", + "rmp-serde", + "serde", + "serde_json", + "thiserror 0.0.1", + "typetag", +] + +[[package]] +name = "daqbuf-items-proc" +version = "0.0.3" +dependencies = [ + "syn", +] + +[[package]] +name = "daqbuf-netpod" +version = "0.0.4" +dependencies = [ + "bytes", + "chrono", + "daqbuf-err", + "futures-util", + "hex", + "http", + "humantime", + "humantime-serde", + "num-traits", + "rand", + "serde", + "serde_json", + "thiserror 0.0.1", + "tracing", + "url", +] + +[[package]] +name = "daqbuf-parse" +version = "0.0.3" +dependencies = [ + "byteorder", + "bytes", + "chrono", + "daqbuf-err", + "daqbuf-netpod", + "hex", + "humantime-serde", + "nom", + "serde", + "serde_json", +] + +[[package]] +name = "daqbuf-query" +version = "0.0.3" +dependencies = [ + "chrono", + "daqbuf-netpod", + "humantime", + "humantime-serde", + "serde", + "serde_json", + "thiserror 0.0.1", + "tracing", + "url", +] + +[[package]] +name = "daqbuf-series" +version = "0.0.2" +dependencies = [ + "serde", +] + +[[package]] +name = "daqbuf-streams" +version = "0.0.3" +dependencies = [ + "arrayref", + "async-channel", + "byteorder", + "bytes", + "chrono", + "ciborium", + "crc32fast", + "daqbuf-items-0", + "daqbuf-items-2", + "daqbuf-netpod", + "daqbuf-parse", + "daqbuf-query", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project", + "rand_xoshiro", + "serde", + "serde_cbor", + "serde_json", + "thiserror 0.0.1", + "typetag", +] + +[[package]] +name = "daqingest" +version = "0.2.5-aa.2" +dependencies = [ + "async-channel", "bytes", "chrono", "clap", + "daqbuf-err", + "daqbuf-netpod", + "daqbuf-series", "dbpg", - "err", "futures-util", "ingest-linux", "libc", "log 0.0.1", "netfetch", - "netpod", "scywr", "serde", - "series", "serieswriter", "stats", "taskrun", @@ -555,7 +724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -569,7 +738,7 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -579,17 +748,17 @@ dependencies = [ name = "dbpg" version = "0.0.1" dependencies = [ - "async-channel 2.3.1", + "async-channel", "batchtools", "chrono", - "err", + "daqbuf-err", + "daqbuf-netpod", + "daqbuf-series", "futures-util", "log 0.0.1", "md-5", - "netpod", "serde", "serde_json", - "series", "stats", "taskrun", "tokio-postgres", @@ -615,6 +784,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.13.0" @@ -649,33 +829,6 @@ dependencies = [ "typeid", ] -[[package]] -name = "err" -version = "0.0.5" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "async-channel 2.3.1", - "backtrace", - "chrono", - "http", - "hyper", - "regex", - "rmp-serde", - "serde", - "serde_cbor", - "serde_json", - "thiserror 0.0.1", - "tokio", - "url", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "event-listener" version = "5.3.1" @@ -689,11 +842,11 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 5.3.1", + "event-listener", "pin-project-lite", ] @@ -709,6 +862,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -720,9 +879,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -735,9 +894,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -745,15 +904,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -762,15 +921,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -779,21 +938,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -830,28 +989,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" - -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "half" @@ -888,6 +1028,17 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "heapless" version = "0.7.17" @@ -908,12 +1059,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -937,9 +1082,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -971,31 +1116,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" - -[[package]] -name = "httpclient" -version = "0.0.2" -dependencies = [ - "async-channel 1.9.0", - "bytes", - "err", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "netpod", - "parse", - "serde", - "serde_json", - "tokio", - "tracing", - "url", -] +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1021,14 +1144,13 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", "http", "http-body", "httparse", @@ -1037,26 +1159,22 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", - "want", ] [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", - "futures-channel", "futures-util", "http", "http-body", "hyper", "pin-project-lite", - "socket2", "tokio", "tower-service", - "tracing", ] [[package]] @@ -1082,6 +1200,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1090,22 +1326,33 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] name = "indexmap" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1130,57 +1377,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "items_0" -version = "0.0.2" -dependencies = [ - "bincode", - "bytes", - "chrono", - "erased-serde", - "err", - "futures-util", - "netpod", - "serde", - "serde_json", - "typetag", -] - -[[package]] -name = "items_2" -version = "0.0.2" -dependencies = [ - "bitshuffle", - "bytes", - "chrono", - "ciborium", - "crc32fast", - "erased-serde", - "err", - "futures-util", - "humantime-serde", - "items_0", - "items_proc", - "netpod", - "num-traits", - "parse", - "postcard", - "rmp-serde", - "serde", - "serde_cbor", - "serde_json", - "taskrun", - "tokio", - "typetag", -] - -[[package]] -name = "items_proc" -version = "0.0.2" -dependencies = [ - "syn", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1192,16 +1388,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1213,9 +1410,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -1294,21 +1497,20 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "mrucache" version = "0.0.1" dependencies = [ - "hashbrown", + "hashbrown 0.15.2", "log 0.0.1", ] @@ -1317,31 +1519,35 @@ name = "netfetch" version = "0.0.3" dependencies = [ "arrayref", - "async-channel 2.3.1", + "async-channel", "axum", "batchtools", "byteorder", "bytes", "chrono", "ciborium", + "daqbuf-ca-proto", + "daqbuf-ca-proto-tokio", + "daqbuf-err", + "daqbuf-items-0", + "daqbuf-items-2", + "daqbuf-netpod", + "daqbuf-series", + "daqbuf-streams", "dashmap 6.1.0", "dbpg", - "err", "futures-util", - "hashbrown", + "hashbrown 0.15.2", "hex", "http-body", "humantime", "humantime-serde", "ingest-linux", - "items_0", - "items_2", "lazy_static", "libc", "log 0.0.1", "md-5", "mrucache", - "netpod", "pin-project", "regex", "scywr", @@ -1350,39 +1556,17 @@ dependencies = [ "serde_helper", "serde_json", "serde_yaml", - "series", "serieswriter", "slidebuf", "smallvec", "stats", - "streams", "taskrun", + "thiserror 0.0.1", "tokio-stream", "tracing", "url", ] -[[package]] -name = "netpod" -version = "0.0.2" -dependencies = [ - "async-channel 1.9.0", - "bytes", - "chrono", - "err", - "futures-util", - "hex", - "http", - "humantime", - "humantime-serde", - "num-traits", - "rand", - "serde", - "serde_json", - "tracing", - "url", -] - [[package]] name = "nom" version = "7.1.3" @@ -1420,18 +1604,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "overload" @@ -1468,23 +1652,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "parse" -version = "0.0.2" -dependencies = [ - "byteorder", - "bytes", - "chrono", - "err", - "hex", - "humantime-serde", - "netpod", - "nom", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "paste" version = "1.0.15" @@ -1517,18 +1684,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", @@ -1537,9 +1704,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -1549,9 +1716,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "postcard" -version = "1.0.10" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" dependencies = [ "cobs", "embedded-io 0.4.0", @@ -1609,30 +1776,13 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] -[[package]] -name = "query" -version = "0.0.1" -dependencies = [ - "chrono", - "err", - "humantime", - "humantime-serde", - "items_0", - "items_2", - "netpod", - "serde", - "serde_json", - "tracing", - "url", -] - [[package]] name = "quote" version = "1.0.37" @@ -1692,18 +1842,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1713,9 +1863,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1724,9 +1874,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rmp" @@ -1767,9 +1917,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -1785,9 +1935,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scylla" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8139623d3fb0c8205b15e84fa587f3aa0ba61f876c19a9157b688f7c1763a7c5" +checksum = "f6075cf39d1b59505fd6777c4028d6b60b2621ea7f79b6b7fb2e02d4c07c4ca9" dependencies = [ "arc-swap", "async-trait", @@ -1796,7 +1946,7 @@ dependencies = [ "chrono", "dashmap 5.5.3", "futures", - "hashbrown", + "hashbrown 0.14.5", "histogram", "itertools", "lazy_static", @@ -1808,7 +1958,7 @@ dependencies = [ "smallvec", "snap", "socket2", - "thiserror 1.0.64", + "thiserror 1.0.69", "tokio", "tracing", "uuid", @@ -1816,9 +1966,9 @@ dependencies = [ [[package]] name = "scylla-cql" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de7020bcd1f6fdbeaed356cd426bf294b2071bd7120d48d2e8e319295e2acdcd" +checksum = "6e032407a921ad7278d58a313829376950a6c81d9680d454cdf7f07891605f3c" dependencies = [ "async-trait", "byteorder", @@ -1826,16 +1976,18 @@ dependencies = [ "lz4_flex", "scylla-macros", "snap", - "thiserror 1.0.64", + "stable_deref_trait", + "thiserror 1.0.69", "tokio", "uuid", + "yoke", ] [[package]] name = "scylla-macros" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3859b6938663fc5062e3b26f3611649c9bd26fb252e85f6fdfa581e0d2ce74b6" +checksum = "9227c539e4a7df86998b949b973567b3569d1ca179216cfc787b918522f128fe" dependencies = [ "darling", "proc-macro2", @@ -1847,16 +1999,16 @@ dependencies = [ name = "scywr" version = "0.0.1" dependencies = [ - "async-channel 2.3.1", + "async-channel", "bytes", - "err", + "daqbuf-err", + "daqbuf-netpod", + "daqbuf-series", "futures-util", "log 0.0.1", - "netpod", "pin-project", "scylla", "serde", - "series", "smallvec", "stackfuture", "stats", @@ -1871,9 +2023,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] @@ -1890,9 +2042,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1908,9 +2060,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -1953,30 +2105,21 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "series" -version = "0.0.2" -dependencies = [ - "err", - "log 0.0.1", - "serde", -] - [[package]] name = "serieswriter" version = "0.0.2" dependencies = [ - "async-channel 2.3.1", + "async-channel", + "daqbuf-err", + "daqbuf-items-0", + "daqbuf-items-2", + "daqbuf-netpod", + "daqbuf-series", "dbpg", - "err", "futures-util", - "items_0", - "items_2", "log 0.0.1", - "netpod", "scywr", "serde", - "series", "smallvec", "stats", "taskrun", @@ -2055,12 +2198,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2116,34 +2259,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "streams" -version = "0.0.2" -dependencies = [ - "arrayref", - "async-channel 1.9.0", - "byteorder", - "bytes", - "chrono", - "ciborium", - "crc32fast", - "err", - "futures-util", - "httpclient", - "items_0", - "items_2", - "netpod", - "parse", - "pin-project", - "query", - "serde", - "serde_cbor", - "serde_json", - "tokio", - "tokio-stream", - "typetag", -] - [[package]] name = "stringprep" version = "0.1.5" @@ -2169,9 +2284,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -2186,9 +2301,20 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "taskrun" @@ -2196,7 +2322,7 @@ version = "0.0.4" dependencies = [ "backtrace", "chrono", - "err", + "daqbuf-err", "futures-util", "time", "tokio", @@ -2215,11 +2341,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.64", + "thiserror-impl 1.0.69", ] [[package]] @@ -2234,9 +2360,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -2255,9 +2381,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -2276,14 +2402,24 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -2301,9 +2437,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -2315,7 +2451,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2368,9 +2504,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -2409,9 +2545,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log 0.4.22", "pin-project-lite", @@ -2421,9 +2557,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -2432,9 +2568,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -2453,9 +2589,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -2466,12 +2602,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "twox-hash" version = "1.6.3" @@ -2520,15 +2650,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -2541,9 +2671,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unsafe-libyaml" @@ -2553,15 +2683,27 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2570,9 +2712,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] @@ -2589,15 +2731,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2612,9 +2745,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", "once_cell", @@ -2623,9 +2756,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log 0.4.22", @@ -2638,9 +2771,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2648,9 +2781,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", @@ -2661,15 +2794,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" dependencies = [ "js-sys", "wasm-bindgen", @@ -2726,6 +2859,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2790,6 +2932,42 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2810,3 +2988,46 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/daqingest/src/tools.rs b/daqingest/src/tools.rs index 848d979..a674300 100644 --- a/daqingest/src/tools.rs +++ b/daqingest/src/tools.rs @@ -11,6 +11,7 @@ use err::ThisError; use futures_util::future; use futures_util::stream; use futures_util::StreamExt; +use futures_util::TryStreamExt; use log::*; use netpod::ttl::RetentionTime; use netpod::Database; @@ -36,6 +37,7 @@ pub enum Error { ScyllaQuery(#[from] QueryError), ScyllaNextRowError(#[from] NextRowError), ScyllaSchema(#[from] scywr::schema::Error), + ScyllaTypeCheck(#[from] scywr::scylla::deserialize::TypeCheckError), ParseError(String), InvalidValue, } @@ -89,14 +91,15 @@ async fn remove_older_series( ) .await?; type RowType = (i64,); - let mut it = it.into_typed::(); - while let Some(e) = it.next().await { - let row = e?; - let ts_msp = row.0; + let mut it = it.rows_stream::()?; + while let Some((ts_msp,)) = it.try_next().await? { debug!("remove ts_msp {}", ts_msp); - let mut it = scy.execute_iter(qu_delete.clone(), (series as i64, ts_msp)).await?; + let mut it = scy + .execute_iter(qu_delete.clone(), (series as i64, ts_msp)) + .await? + .rows_stream::<()>()?; let mut j = 0; - while let Some(_) = it.next().await { + while let Some(_) = it.try_next().await? { j += 1; } debug!("rows returned {}", j); @@ -182,12 +185,11 @@ pub async fn remove_older_all_rt(ts_cut: TsMs, ks: &str, rt: RetentionTime, scy: let stmts = Stmts::new(ks, rt.clone(), &scy).await?; type RowType = (i64,); let it = scy.execute_iter(stmts.qu_select_series.as_ref().clone(), ()).await?; - let mut it = it.into_typed::(); + let mut it = it.rows_stream::()?; let mut series_ids = Vec::with_capacity(1000000); let print_dt = Duration::from_millis(2000); let mut print_next = Instant::now() + print_dt; - while let Some(e) = it.next().await { - let row = e?; + while let Some(row) = it.try_next().await? { let series = SeriesId::new(row.0 as u64); series_ids.push(series); let tsnow = Instant::now(); @@ -219,14 +221,13 @@ async fn remove_older_all_series(ts_cut: TsMs, series: SeriesId, stmts: &Stmts, let mut it = scy .execute_iter(stmts.qu_select_msp.clone(), (series.to_i64(),)) .await? - .into_typed::(); + .rows_stream::()?; let mut msp_last = 0; let mut to_remove = Vec::new(); let mut n_keep = 0; let mut n_remove = 0; let ts2 = Instant::now(); - while let Some(e) = it.next().await { - let row = e?; + while let Some(row) = it.try_next().await? { let msp = row.0 as u64; if msp < msp_last { panic!("msp ordering error {:?}", series); @@ -319,10 +320,9 @@ pub async fn find_older_msp( let mut it = scy .execute_iter(qu.clone(), (trbeg, trend)) .await? - .into_typed::<(i64, i64)>(); + .rows_stream::<(i64, i64)>()?; let mut c = 0; - while let Some(u) = it.next().await { - let row = u?; + while let Some(row) = it.try_next().await? { let series = row.0 as u64; let ts_msp = row.1 as u64; if series == 9033627543553833740 { diff --git a/mrucache/Cargo.toml b/mrucache/Cargo.toml index 4abf57b..7a80b61 100644 --- a/mrucache/Cargo.toml +++ b/mrucache/Cargo.toml @@ -5,5 +5,5 @@ edition = "2021" authors = ["Dominik Werder "] [dependencies] -hashbrown = "0.14" +hashbrown = "0.15.2" log = { path = "../log" } diff --git a/netfetch/Cargo.toml b/netfetch/Cargo.toml index 618499b..1c99afe 100644 --- a/netfetch/Cargo.toml +++ b/netfetch/Cargo.toml @@ -34,7 +34,7 @@ lazy_static = "1" libc = "0.2" slidebuf = "0.0.1" dashmap = "6.0.1" -hashbrown = "0.14.3" +hashbrown = "0.15.2" smallvec = "1.13.2" thiserror = "=0.0.1" log = { path = "../log" } diff --git a/netfetch/src/metrics/delete.rs b/netfetch/src/metrics/delete.rs index 0286e37..e0458ec 100644 --- a/netfetch/src/metrics/delete.rs +++ b/netfetch/src/metrics/delete.rs @@ -11,6 +11,7 @@ use core::fmt; use err::thiserror; use err::ThisError; use futures_util::StreamExt; +use futures_util::TryStreamExt; use netpod::log::*; use netpod::ttl::RetentionTime; use netpod::ScalarType; @@ -57,9 +58,9 @@ pub enum Error { MissingEndDate, ScyllaTransport(#[from] scylla::transport::errors::NewSessionError), ScyllaQuery(#[from] scylla::transport::errors::QueryError), - ScyllaRowType(#[from] scylla::transport::query_result::RowsExpectedError), ScyllaRowError(#[from] scylla::cql_to_rust::FromRowError), ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError), + ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError), InvalidTimestamp, } @@ -157,9 +158,8 @@ async fn delete_try( let mut it = scy .execute_iter(qu.clone(), (series.to_i64(),)) .await? - .into_typed::<(i64,)>(); - while let Some(x) = it.next().await { - let (msp,) = x?; + .rows_stream::<(i64,)>()?; + while let Some((msp,)) = it.try_next().await? { let msp = TsMs::from_ms_u64(msp as _); let msp_ns = msp.ns_u64(); delete_val(series.clone(), msp, beg, end, &qu_delete_val, &scy).await?; @@ -191,10 +191,8 @@ async fn delete_val( let mut it = scy .execute_iter(qu_delete_val.clone(), params) .await? - .into_typed::<(i64,)>(); - while let Some(x) = it.next().await { - let (lsp,) = x?; - } + .rows_stream::<(i64,)>()?; + while let Some((lsp,)) = it.try_next().await? {} Ok(()) } diff --git a/netfetch/src/metrics/ingest.rs b/netfetch/src/metrics/ingest.rs index 853c93c..4b0892a 100644 --- a/netfetch/src/metrics/ingest.rs +++ b/netfetch/src/metrics/ingest.rs @@ -10,11 +10,8 @@ use err::thiserror; use err::ThisError; use futures_util::StreamExt; use futures_util::TryStreamExt; -use items_2::eventsdim0::EventsDim0; -use items_2::eventsdim0::EventsDim0NoPulse; -use items_2::eventsdim0enum::EventsDim0Enum; -use items_2::eventsdim1::EventsDim1; -use items_2::eventsdim1::EventsDim1NoPulse; +use items_2::binning::container_events::ContainerEvents; +use items_2::binning::container_events::EventValueType; use netpod::log::*; use netpod::ttl::RetentionTime; use netpod::EnumVariant; @@ -283,7 +280,7 @@ async fn post_v01_try( } ScalarType::STRING => { evpush_dim0::(&frame, deque, &mut writer, |x| { - DataValue::Scalar(ScalarValue::String(x)) + DataValue::Scalar(ScalarValue::String(x.into())) })?; } ScalarType::Enum => { @@ -351,22 +348,20 @@ fn evpush_dim0( f1: F1, ) -> Result<(), Error> where - T: for<'a> Deserialize<'a> + fmt::Debug + Clone, - F1: Fn(T) -> DataValue, + T: EventValueType, + F1: Fn(::IterTy1<'_>) -> DataValue, { - let evs: EventsDim0NoPulse = ciborium::de::from_reader(Cursor::new(frame)) + let evs: ContainerEvents = ciborium::de::from_reader(Cursor::new(frame)) .map_err(|e| { error!("cbor decode error {e}"); }) .map_err(|_| Error::Decode)?; - let evs: EventsDim0 = evs.into(); // trace_input!("see events {:?}", evs); let stnow = SystemTime::now(); let tsev = TsNano::from_system_time(stnow); let tsnow = Instant::now(); let mut emit_state = WritableTypeState::new(writer.sid()); - for (i, (&ts, val)) in evs.tss.iter().zip(evs.values.iter()).enumerate() { - let ts = TsNano::from_ns(ts); + for (i, (ts, val)) in evs.iter_zip().enumerate() { let val = val.clone(); trace_input!("ev {:6} {:20} {:20?}", i, ts, val); let val = f1(val); @@ -381,7 +376,7 @@ fn evpush_dim0_enum( deque: &mut VecDeque, writer: &mut ValueSeriesWriter, ) -> Result<(), Error> { - let evs: EventsDim0Enum = ciborium::de::from_reader(Cursor::new(frame)) + let evs: ContainerEvents = ciborium::de::from_reader(Cursor::new(frame)) .map_err(|e| { error!("cbor decode error {e}"); }) @@ -391,17 +386,10 @@ fn evpush_dim0_enum( let tsev = TsNano::from_system_time(stnow); let tsnow = Instant::now(); let mut emit_state = WritableTypeState::new(writer.sid()); - for (i, ((&ts, val), vals)) in evs - .tss - .iter() - .zip(evs.values.iter()) - .zip(evs.valuestrs.iter()) - .enumerate() - { - let ts = TsNano::from_ns(ts); + for (i, (ts, val)) in evs.iter_zip().enumerate() { let val = val.clone(); trace_input!("ev {:6} {:20} {:20?}", i, ts, val); - let val = DataValue::Scalar(ScalarValue::Enum(val as i16, vals.clone())); + let val = DataValue::Scalar(ScalarValue::Enum(val.ix as i16, val.name.into())); writer.write(WritableType(ts, val), &mut emit_state, tsnow, tsev, deque)?; } Ok(()) @@ -414,23 +402,21 @@ fn evpush_dim1( f1: F1, ) -> Result<(), Error> where - T: for<'a> Deserialize<'a> + fmt::Debug + Clone, - F1: Fn(Vec) -> DataValue, + Vec: EventValueType, + F1: Fn( as EventValueType>::IterTy1<'_>) -> DataValue, { - let evs: EventsDim1NoPulse = ciborium::de::from_reader(Cursor::new(frame)) + let evs: ContainerEvents> = ciborium::de::from_reader(Cursor::new(frame)) .map_err(|e| { error!("cbor decode error {e}"); }) .map_err(|_| Error::Decode)?; - let evs: EventsDim1 = evs.into(); trace_input!("see events {:?}", evs); warn!("TODO require timestamp in input format"); let stnow = SystemTime::now(); let tsev = TsNano::from_system_time(stnow); let tsnow = Instant::now(); let mut emit_state = WritableTypeState::new(writer.sid()); - for (i, (&ts, val)) in evs.tss.iter().zip(evs.values.iter()).enumerate() { - let ts = TsNano::from_ns(ts); + for (i, (ts, val)) in evs.iter_zip().enumerate() { let val = val.clone(); trace_input!("ev {:6} {:20} {:20?}", i, ts, val); let val = f1(val); diff --git a/scywr/Cargo.toml b/scywr/Cargo.toml index 6159c01..524d833 100644 --- a/scywr/Cargo.toml +++ b/scywr/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] futures-util = "0.3.28" async-channel = "2.3.1" -scylla = "0.14.0" +scylla = "0.15.0" smallvec = "1.11.0" pin-project = "1.1.5" stackfuture = "0.3.0" diff --git a/scywr/src/err.rs b/scywr/src/err.rs index d37d45b..62231b3 100644 --- a/scywr/src/err.rs +++ b/scywr/src/err.rs @@ -21,7 +21,7 @@ pub trait IntoSimplerError { impl IntoSimplerError for QueryError { fn into_simpler(self) -> Error { - let e = self; + let e = &self; match e { QueryError::DbError(e, msg) => match e { DbError::Unavailable { .. } => Error::DbUnavailable, @@ -31,16 +31,8 @@ impl IntoSimplerError for QueryError { DbError::WriteTimeout { .. } => Error::DbTimeout, _ => Error::DbError(format!("{e} {msg}")), }, - QueryError::BadQuery(e) => Error::DbError(e.to_string()), - QueryError::IoError(e) => Error::DbError(e.to_string()), - QueryError::ProtocolError(e) => Error::DbError(e.to_string()), - QueryError::InvalidMessage(e) => Error::DbError(e.to_string()), QueryError::TimeoutError => Error::DbTimeout, - QueryError::TooManyOrphanedStreamIds(e) => Error::DbError(e.to_string()), - QueryError::UnableToAllocStreamId => Error::DbError(e.to_string()), - QueryError::RequestTimeout(e) => Error::DbError(e.to_string()), - QueryError::TranslationError(e) => Error::DbError(e.to_string()), - QueryError::CqlResponseParseError(e) => Error::DbError(e.to_string()), + _ => Error::DbError(e.to_string()), } } } diff --git a/scywr/src/iteminsertqueue.rs b/scywr/src/iteminsertqueue.rs index a65c25d..d794e3b 100644 --- a/scywr/src/iteminsertqueue.rs +++ b/scywr/src/iteminsertqueue.rs @@ -663,7 +663,7 @@ impl InsertFut { Self { scy, qu, - fut: Box::pin(async { Err(QueryError::InvalidMessage("no longer used".into())) }), + fut: Box::pin(async { Err(QueryError::TimeoutError) }), } } } diff --git a/scywr/src/schema.rs b/scywr/src/schema.rs index 4c7c2b0..6c72e92 100644 --- a/scywr/src/schema.rs +++ b/scywr/src/schema.rs @@ -4,11 +4,10 @@ use crate::session::ScySession; use err::thiserror; use err::ThisError; use futures_util::StreamExt; +use futures_util::TryStreamExt; use log::*; use netpod::ttl::RetentionTime; use scylla::transport::errors::DbError; -use scylla::transport::errors::QueryError; -use scylla::transport::iterator::NextRowError; use std::collections::BTreeMap; use std::fmt; use std::time::Duration; @@ -18,9 +17,10 @@ use std::time::Duration; pub enum Error { NoKeyspaceChosen, Fmt(#[from] fmt::Error), - Query(#[from] QueryError), + Query(#[from] scylla::transport::errors::QueryError), NewSession(String), - ScyllaNextRow(#[from] NextRowError), + ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError), + ScyllaTypecheck(#[from] scylla::deserialize::TypeCheckError), MissingData, AddColumnImpossible, BadSchema, @@ -87,13 +87,10 @@ impl Changeset { pub async fn has_keyspace(name: &str, scy: &ScySession) -> Result { let cql = "select keyspace_name from system_schema.keyspaces where keyspace_name = ?"; - let mut res = scy.query_iter(cql, (name,)).await?; - while let Some(k) = res.next().await { - let row = k?; - if let Some(table_name) = row.columns[0].as_ref().unwrap().as_text() { - if table_name == name { - return Ok(true); - } + let mut res = scy.query_iter(cql, (name,)).await?.rows_stream::<(String,)>()?; + while let Some((table_name,)) = res.try_next().await? { + if table_name == name { + return Ok(true); } } Ok(false) @@ -102,19 +99,17 @@ pub async fn has_keyspace(name: &str, scy: &ScySession) -> Result { pub async fn has_table(name: &str, scy: &ScySession) -> Result { let cql = "select table_name from system_schema.tables where keyspace_name = ?"; let ks = scy.get_keyspace().ok_or_else(|| Error::NoKeyspaceChosen)?; - let mut res = scy.query_iter(cql, (ks.as_ref(),)).await?; - while let Some(k) = res.next().await { - let row = k?; - if let Some(table_name) = row.columns[0].as_ref().unwrap().as_text() { - if table_name == name { - return Ok(true); - } + let mut res = scy.query_iter(cql, (ks.as_ref(),)).await?.rows_stream::<(String,)>()?; + while let Some((table_name,)) = res.try_next().await? { + if table_name == name { + return Ok(true); } } Ok(false) } pub async fn check_table_readable(name: &str, scy: &ScySession) -> Result { + use crate::scylla::transport::errors::QueryError; match scy.query_unpaged(format!("select * from {} limit 1", name), ()).await { Ok(_) => Ok(true), Err(e) => match &e { @@ -318,7 +313,8 @@ impl GenTwcsTab { " from system_schema.tables where keyspace_name = ? and table_name = ?" ); let x = scy.query_iter(cql, (self.keyspace(), self.name())).await?; - let mut it = x.into_typed::<(i32, i32, BTreeMap)>(); + let mut it = x.rows_stream::<(i32, i32, BTreeMap)>()?; + // let mut it = x.into_typed::<(i32, i32, BTreeMap)>(); let mut rows = Vec::new(); while let Some(u) = it.next().await { let row = u?; @@ -369,13 +365,12 @@ impl GenTwcsTab { let mut it = scy .query_iter(cql, (self.keyspace(), self.name())) .await? - .into_typed::<(String, String)>(); + .rows_stream::<(String, String)>()?; let mut names_exist = Vec::new(); let mut types_exist = Vec::new(); - while let Some(x) = it.next().await { - let row = x?; - names_exist.push(row.0); - types_exist.push(row.1); + while let Some((name, ty)) = it.try_next().await? { + names_exist.push(name); + types_exist.push(ty); } debug!("names_exist {:?} types_exist {:?}", names_exist, types_exist); for (cn, ct) in self.col_names.iter().zip(self.col_types.iter()) { @@ -433,13 +428,12 @@ async fn get_columns(keyspace: &str, table: &str, scy: &ScySession) -> Result(); - while let Some(x) = it.next().await { - let row = x?; + .rows_stream::<(String, String, String, i32, String)>()?; + while let Some((name, ..)) = it.try_next().await? { // columns: // column_name (text) // type (text): text, blob, int, ... - ret.push(row.0); + ret.push(name); } Ok(ret) } diff --git a/scywr/src/tools.rs b/scywr/src/tools.rs index 46b950d..14c079e 100644 --- a/scywr/src/tools.rs +++ b/scywr/src/tools.rs @@ -1,6 +1,6 @@ use crate::config::ScyllaIngestConfig; use crate::session::create_session; -use futures_util::StreamExt; +use futures_util::TryStreamExt; use log::*; use scylla::transport::errors::NewSessionError; use scylla::transport::errors::QueryError; @@ -25,6 +25,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: scylla::deserialize::TypeCheckError) -> Self { + Self(err::Error::with_msg_no_trace(format!("{e:?}"))) + } +} + pub async fn list_pkey(scylla_conf: &ScyllaIngestConfig) -> Result<(), Error> { let scy = create_session(scylla_conf) .await @@ -39,17 +45,13 @@ pub async fn list_pkey(scylla_conf: &ScyllaIngestConfig) -> Result<(), Error> { let t2 = if t1 < i64::MAX - td { t1 + td } else { i64::MAX }; let pct = (t1 - i64::MIN) as u64 / (u64::MAX / 100000); info!("Token range {:.2}%", pct as f32 * 1e-3); - let mut it = scy.execute_iter(query.clone(), (t1, t2)).await?; - while let Some(x) = it.next().await { - let r = x?; - if r.columns.len() < 2 { - warn!("see {} columns", r.columns.len()); - } else { - let pulse_a_token = r.columns[0].as_ref().unwrap().as_bigint().unwrap(); - let pulse_a = r.columns[1].as_ref().unwrap().as_bigint().unwrap(); - info!("pulse_a_token {pulse_a_token} pulse_a {pulse_a}"); - pulse_a_max = pulse_a_max.max(pulse_a); - } + let mut it = scy + .execute_iter(query.clone(), (t1, t2)) + .await? + .rows_stream::<(i64, i64)>()?; + while let Some((pulse_a_token, pulse_a)) = it.try_next().await? { + info!("pulse_a_token {pulse_a_token} pulse_a {pulse_a}"); + pulse_a_max = pulse_a_max.max(pulse_a); } if t2 == i64::MAX { info!("end of token range"); @@ -75,18 +77,12 @@ pub async fn list_pulses(scylla_conf: &ScyllaIngestConfig) -> Result<(), Error> let t2 = if t1 < i64::MAX - td { t1 + td } else { i64::MAX }; let pct = (t1 - i64::MIN) as u64 / (u64::MAX / 100000); info!("Token range {:.2}%", pct as f32 * 1e-3); - let mut it = scy.execute_iter(query.clone(), (t1, t2)).await?; - while let Some(x) = it.next().await { - let r = x?; - if r.columns.len() < 2 { - warn!("see {} columns", r.columns.len()); - } else { - let tsa_token = r.columns[0].as_ref().unwrap().as_bigint().unwrap(); - let tsa = r.columns[1].as_ref().unwrap().as_int().unwrap() as u32; - let tsb = r.columns[2].as_ref().unwrap().as_int().unwrap() as u32; - let pulse = r.columns[3].as_ref().unwrap().as_bigint().unwrap() as u64; - info!("tsa_token {tsa_token:21} tsa {tsa:12} tsb {tsb:12} pulse {pulse:21}"); - } + let mut it = scy + .execute_iter(query.clone(), (t1, t2)) + .await? + .rows_stream::<(i64, i32, i32, i64)>()?; + while let Some((tsa_token, tsa, tsb, pulse)) = it.try_next().await? { + info!("tsa_token {tsa_token:21} tsa {tsa:12} tsb {tsb:12} pulse {pulse:21}"); } if t2 == i64::MAX { info!("end of token range"); @@ -110,20 +106,13 @@ pub async fn fetch_events(backend: &str, channel: &str, scylla_conf: &ScyllaInge ) .await?; let mut rowcnt = 0; - let mut it = scy.execute_iter(qu_series.clone(), (backend, channel)).await?; - while let Some(x) = it.next().await { - let r = x?; - info!("Got row: {r:?}"); + let mut it = scy + .execute_iter(qu_series.clone(), (backend, channel)) + .await? + .rows_stream::<(i64, i32, i32, i64)>()?; + while let Some((tsa_token, tsa, tsb, pulse)) = it.try_next().await? { if false { - if r.columns.len() < 2 { - warn!("see {} columns", r.columns.len()); - } else { - let tsa_token = r.columns[0].as_ref().unwrap().as_bigint().unwrap(); - let tsa = r.columns[1].as_ref().unwrap().as_int().unwrap() as u32; - let tsb = r.columns[2].as_ref().unwrap().as_int().unwrap() as u32; - let pulse = r.columns[3].as_ref().unwrap().as_bigint().unwrap() as u64; - info!("tsa_token {tsa_token:21} tsa {tsa:12} tsb {tsb:12} pulse {pulse:21}"); - } + info!("tsa_token {tsa_token:21} tsa {tsa:12} tsb {tsb:12} pulse {pulse:21}"); } rowcnt += 1; } diff --git a/serieswriter/src/binwritergrid.rs b/serieswriter/src/binwritergrid.rs index ede6a99..179def3 100644 --- a/serieswriter/src/binwritergrid.rs +++ b/serieswriter/src/binwritergrid.rs @@ -86,13 +86,13 @@ impl BinWriterGrid { Ok(()) } - fn handle_output_ready(&mut self, out: ContainerBins, iqdqs: &mut InsertDeques) -> Result<(), Error> { + fn handle_output_ready(&mut self, out: ContainerBins, iqdqs: &mut InsertDeques) -> Result<(), Error> { let selfname = "handle_output_ready"; trace_tick!("{selfname} bins ready len {}", out.len()); for e in out.iter_debug() { trace_tick_verbose!("{e:?}"); } - for (((((((&ts1, &ts2), &cnt), &min), &max), &avg), &lst), &fnl) in out.zip_iter() { + for (((((((&ts1, &ts2), &cnt), min), max), avg), lst), &fnl) in out.zip_iter() { if fnl == false { info!("non final bin"); } else if cnt == 0 {