From 03e8ac7a70eb14e34e0381ff25d925ea700c473a Mon Sep 17 00:00:00 2001 From: Dominik Werder Date: Wed, 12 Jun 2024 13:43:28 +0200 Subject: [PATCH] Udate deps, backend list for plain node --- .cargo/cargo-lock | 935 ++++++++++++++++---- crates/daqbuffer/src/bin/daqbuffer.rs | 2 +- crates/dbconn/src/channelinfo.rs | 51 +- crates/disk/src/decode.rs | 2 + crates/httpret/src/api1.rs | 13 +- crates/httpret/src/api4.rs | 1 + crates/httpret/src/api4/accounting.rs | 111 ++- crates/httpret/src/api4/backend.rs | 55 ++ crates/httpret/src/api4/eventdata.rs | 13 + crates/httpret/src/api4/events.rs | 66 +- crates/httpret/src/httpret.rs | 12 +- crates/httpret/src/proxy.rs | 3 + crates/httpret/src/proxy/api4/backend.rs | 3 + crates/items_0/src/items_0.rs | 4 +- crates/items_0/src/scalar_ops.rs | 48 + crates/items_0/src/subfr.rs | 6 + crates/items_2/src/channelevents.rs | 8 +- crates/items_2/src/empty.rs | 3 + crates/items_2/src/eventsdim0.rs | 39 +- crates/items_2/src/eventsdim1.rs | 28 +- crates/items_2/src/eventsxbindim0.rs | 28 +- crates/netpod/src/netpod.rs | 139 ++- crates/nodenet/src/conn/test.rs | 3 +- crates/parse/src/api1_parse.rs | 2 + crates/query/src/api4/binned.rs | 5 + crates/query/src/api4/events.rs | 24 +- crates/scyllaconn/src/accounting/toplist.rs | 8 +- crates/scyllaconn/src/events.rs | 86 +- crates/scyllaconn/src/worker.rs | 32 +- crates/streams/src/collect.rs | 5 +- crates/streams/src/plaineventsjson.rs | 8 +- crates/streams/src/plaineventsstream.rs | 1 + crates/streams/src/tcprawclient.rs | 3 +- crates/streams/src/timebinnedjson.rs | 1 + crates/taskrun/src/taskrun.rs | 54 +- 35 files changed, 1459 insertions(+), 343 deletions(-) create mode 100644 crates/httpret/src/api4/backend.rs diff --git a/.cargo/cargo-lock b/.cargo/cargo-lock index b646fa8..5b3d431 100644 --- a/.cargo/cargo-lock +++ b/.cargo/cargo-lock @@ -4,11 +4,11 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ - "gimli 0.28.1", + "gimli 0.29.0", ] [[package]] @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -136,9 +136,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arc-swap" @@ -200,7 +200,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -211,7 +211,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -223,6 +223,12 @@ dependencies = [ "critical-section", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -242,7 +248,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -276,9 +282,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -405,12 +411,24 @@ name = "bytes" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +dependencies = [ + "serde", +] [[package]] name = "cc" -version = "1.0.97" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" [[package]] name = "cfg-if" @@ -462,9 +480,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive", @@ -472,33 +490,33 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "cobs" @@ -691,9 +709,9 @@ checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -719,9 +737,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -756,9 +774,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -828,14 +846,38 @@ dependencies = [ "url", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.9", + "darling_macro 0.20.9", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -848,8 +890,19 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn 2.0.64", + "strsim 0.11.1", + "syn 2.0.66", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", ] [[package]] @@ -858,9 +911,9 @@ version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ - "darling_core", + "darling_core 0.20.9", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -917,6 +970,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.7" @@ -967,6 +1051,26 @@ dependencies = [ "url", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "document-features" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5282ad69563b5fc40319526ba27e0e7363d552a896f0297d54f767717f9b95" +dependencies = [ + "litrs", +] + [[package]] name = "dq" version = "0.1.0" @@ -984,6 +1088,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + [[package]] name = "either" version = "1.12.0" @@ -1031,10 +1141,10 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08b6c6ab82d70f08844964ba10c7babb716de2ecaeab9be5717918a5177d3af" dependencies = [ - "darling", + "darling 0.20.9", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1074,6 +1184,16 @@ dependencies = [ "url", ] +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -1082,9 +1202,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", @@ -1097,7 +1217,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -1108,10 +1228,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] -name = "finl_unicode" -version = "1.2.0" +name = "fastrand" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] [[package]] name = "flate2" @@ -1210,7 +1342,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1264,13 +1396,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1286,9 +1420,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "h2" @@ -1311,15 +1445,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap 2.2.6", "slab", @@ -1481,12 +1615,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -1494,9 +1628,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpclient" @@ -1584,9 +1718,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -1615,7 +1749,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -1633,7 +1767,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1641,9 +1775,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -1682,6 +1816,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1690,12 +1942,14 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", ] [[package]] @@ -1706,6 +1960,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1716,6 +1971,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.5", + "serde", ] [[package]] @@ -1778,7 +2034,7 @@ dependencies = [ name = "items_proc" version = "0.0.2" dependencies = [ - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -1828,9 +2084,27 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" @@ -1947,9 +2221,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -2054,7 +2328,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2078,9 +2352,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -2105,9 +2379,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -2190,7 +2464,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2287,18 +2561,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -2306,22 +2580,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -2468,14 +2742,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -2489,13 +2763,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -2506,9 +2780,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "region" @@ -2598,6 +2872,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -2610,6 +2897,31 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schemars" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "schemars_derive" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.66", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2641,7 +2953,7 @@ dependencies = [ "smallvec", "snap", "socket2", - "thiserror 1.0.60", + "thiserror 1.0.61", "tokio", "tracing", "uuid", @@ -2659,7 +2971,7 @@ dependencies = [ "lz4_flex", "scylla-macros", "snap", - "thiserror 1.0.60", + "thiserror 1.0.61", "tokio", "uuid", ] @@ -2670,10 +2982,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e5fe1d389adebe6a1a27bce18b81a65ff18c25d58a795de490e18b0e7a27b9f" dependencies = [ - "darling", + "darling 0.20.9", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -2707,12 +3019,15 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "serde" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -2740,13 +3055,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] @@ -2760,6 +3086,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -2914,15 +3249,21 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2948,9 +3289,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.64" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad3dee41f36859875573074334c200d1add8e4a87bb37113ebd31d926b7b11f" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2963,12 +3304,34 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.14" @@ -2991,6 +3354,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "0.0.1" @@ -3001,11 +3376,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl 1.0.60", + "thiserror-impl 1.0.61", ] [[package]] @@ -3015,18 +3390,18 @@ source = "git+https://github.com/dominikwerder/thiserror.git#052df05c18b5f26b462 dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -3079,6 +3454,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -3096,9 +3481,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -3126,13 +3511,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -3185,6 +3570,65 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.22.14", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.6.13", +] + [[package]] name = "tonic" version = "0.10.2" @@ -3199,7 +3643,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -3250,7 +3694,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3264,7 +3707,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -3368,7 +3811,7 @@ checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", ] [[package]] @@ -3392,6 +3835,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -3400,20 +3849,33 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] -name = "utf8parse" -version = "0.2.1" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -3478,7 +3940,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -3500,7 +3962,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3513,9 +3975,9 @@ checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasmer" -version = "4.2.8" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4014573f108a246858299eb230031e268316fd57207bd2e8afc79b20fc7ce983" +checksum = "b1852ee143a2d8143265bfee017c43bf690702d6c2b45a763a2f13e669f5b7ec" dependencies = [ "bytes", "cfg-if", @@ -3528,7 +3990,7 @@ dependencies = [ "serde-wasm-bindgen", "shared-buffer", "target-lexicon", - "thiserror 1.0.60", + "thiserror 1.0.61", "tracing", "wasm-bindgen", "wasmer-compiler", @@ -3541,9 +4003,9 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "4.2.8" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a77bfe259f08e8ec9e77f8f772ebfb4149f799d1f637231c5a5a6a90c447256" +checksum = "6b4f157d715f3bb60c2c9d7b9e48299a30e9209f87f4484f79f9cd586b40b6ee" dependencies = [ "backtrace", "bytes", @@ -3559,18 +4021,19 @@ dependencies = [ "self_cell", "shared-buffer", "smallvec", - "thiserror 1.0.60", + "thiserror 1.0.61", "wasmer-types", "wasmer-vm", "wasmparser", "winapi", + "xxhash-rust", ] [[package]] name = "wasmer-compiler-cranelift" -version = "4.2.8" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9280c47ebc754f95357745a38a995dd766f149e16b26e1b7e35741eb23c03d12" +checksum = "eb457e66b77ca2188fbbd6c2056ec6e8ccb4bddee73e60ba9d39733d7b2e8068" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -3586,10 +4049,32 @@ dependencies = [ ] [[package]] -name = "wasmer-derive" -version = "4.2.8" +name = "wasmer-config" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9352877c4f07fc59146d21b56ae6dc469caf342587f49c81b4fbeafead31972" +checksum = "54a0f70c177b1c5062cfe0f5308c3317751796fef9403c22a0cd7b4cacd4ccd8" +dependencies = [ + "anyhow", + "bytesize", + "derive_builder", + "hex", + "indexmap 2.2.6", + "schemars", + "semver", + "serde", + "serde_cbor", + "serde_json", + "serde_yaml", + "thiserror 1.0.61", + "toml 0.8.14", + "url", +] + +[[package]] +name = "wasmer-derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cd5732ff64370e98986f9753cce13b91cc9d3c4b649e31b0d08d5db69164ea" dependencies = [ "proc-macro-error", "proc-macro2", @@ -3599,25 +4084,30 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "4.2.8" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "749214b6170f2b2fbbfe5b7e7f8d381e64930ac4122f3abceb33cde0292d45d2" +checksum = "c890fd0dbda40df03977b899d1ad7113deba3c225f2cc7b88deb7633044d3e07" dependencies = [ "bytecheck", "enum-iterator", "enumset", + "getrandom", + "hex", "indexmap 1.9.3", "more-asserts", "rkyv", + "sha2", "target-lexicon", - "thiserror 1.0.60", + "thiserror 1.0.61", + "webc", + "xxhash-rust", ] [[package]] name = "wasmer-vm" -version = "4.2.8" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300215479de0deeb453e95aeb1b9c8ffd9bc7d9bd27c5f9e8a184e54db4d31a9" +checksum = "5e0dc60ab800cf0bd44e2d35d88422d256d2470b00c72778f91bfb826c42dbd0" dependencies = [ "backtrace", "cc", @@ -3636,7 +4126,7 @@ dependencies = [ "more-asserts", "region", "scopeguard", - "thiserror 1.0.60", + "thiserror 1.0.61", "wasmer-types", "winapi", ] @@ -3662,6 +4152,35 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webc" +version = "6.0.0-rc1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fc686c7b43c9bc630a499f6ae1f0a4c4bd656576a53ae8a147b0cc9bc983ad" +dependencies = [ + "anyhow", + "base64", + "bytes", + "cfg-if", + "document-features", + "flate2", + "indexmap 1.9.3", + "libc", + "once_cell", + "semver", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "shared-buffer", + "tar", + "tempfile", + "thiserror 1.0.61", + "toml 0.7.8", + "url", + "wasmer-config", +] + [[package]] name = "whoami" version = "1.5.1" @@ -3886,6 +4405,36 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -3895,6 +4444,47 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -3912,5 +4502,48 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.66", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] diff --git a/crates/daqbuffer/src/bin/daqbuffer.rs b/crates/daqbuffer/src/bin/daqbuffer.rs index 132cd59..b3ce06f 100644 --- a/crates/daqbuffer/src/bin/daqbuffer.rs +++ b/crates/daqbuffer/src/bin/daqbuffer.rs @@ -64,7 +64,7 @@ async fn go() -> Result<(), Error> { }; match opts.subcmd { SubCmd::Retrieval(subcmd) => { - info!("daqbuffer version {} +0003", clap::crate_version!()); + info!("daqbuffer version {} +0005", clap::crate_version!()); info!(" service_version {}", service_version); if false { #[allow(non_snake_case)] diff --git a/crates/dbconn/src/channelinfo.rs b/crates/dbconn/src/channelinfo.rs index b952c48..2433ed4 100644 --- a/crates/dbconn/src/channelinfo.rs +++ b/crates/dbconn/src/channelinfo.rs @@ -19,7 +19,7 @@ pub struct ChannelInfo { pub kind: u16, } -pub async fn info_for_series_ids(series_ids: &[u64], pg: &Client) -> Result, Error> { +pub async fn info_for_series_ids(series_ids: &[u64], pg: &Client) -> Result)>, Error> { let (ord, seriess) = series_ids .iter() .enumerate() @@ -34,33 +34,38 @@ pub async fn info_for_series_ids(series_ids: &[u64], pg: &Client) -> Result = row.get(5); - let kind: i16 = row.get(6); - let series = series as u64; - let scalar_type = ScalarType::from_scylla_i32(scalar_type).map_err(|_| Error::BadValue)?; - let shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadValue)?; - let kind = kind as u16; - let e = ChannelInfo { - series, - backend, - name: channel, - scalar_type, - shape, - kind, - }; - ret.push(e); + let series = row.get::<_, i64>(1) as u64; + let backend: Option = row.get(2); + if let Some(backend) = backend { + let channel: String = row.get(3); + let scalar_type: i32 = row.get(4); + let shape_dims: Vec = row.get(5); + let kind: i16 = row.get(6); + let scalar_type = ScalarType::from_scylla_i32(scalar_type).map_err(|_| Error::BadValue)?; + let shape = Shape::from_scylla_shape_dims(&shape_dims).map_err(|_| Error::BadValue)?; + let kind = kind as u16; + let e = ChannelInfo { + series, + backend, + name: channel, + scalar_type, + shape, + kind, + }; + ret.push((series, Some(e))); + } else { + ret.push((series, None)); + } } Ok(ret) } diff --git a/crates/disk/src/decode.rs b/crates/disk/src/decode.rs index 76f76cb..ca93602 100644 --- a/crates/disk/src/decode.rs +++ b/crates/disk/src/decode.rs @@ -269,6 +269,7 @@ fn make_scalar_conv( ScalarType::F64 => ValueDim0FromBytesImpl::::boxed(), ScalarType::BOOL => ValueDim0FromBytesImpl::::boxed(), ScalarType::STRING => ValueDim0FromBytesImpl::::boxed(), + ScalarType::Enum => ValueDim0FromBytesImpl::::boxed(), ScalarType::ChannelStatus => ValueDim0FromBytesImpl::::boxed(), }, Shape::Wave(_) => { @@ -286,6 +287,7 @@ fn make_scalar_conv( ScalarType::F64 => ValueDim1FromBytesImpl::::boxed(shape), ScalarType::BOOL => ValueDim1FromBytesImpl::::boxed(shape), ScalarType::STRING => ValueDim1FromBytesImpl::::boxed(shape), + ScalarType::Enum => ValueDim1FromBytesImpl::::boxed(shape), ScalarType::ChannelStatus => ValueDim1FromBytesImpl::::boxed(shape), } } diff --git a/crates/httpret/src/api1.rs b/crates/httpret/src/api1.rs index 9818809..9b7b811 100644 --- a/crates/httpret/src/api1.rs +++ b/crates/httpret/src/api1.rs @@ -766,7 +766,8 @@ impl DataApiPython3DataStream { self.range.clone().into(), TransformQuery::for_event_blobs(), ); - let subq = EventsSubQuery::from_parts(select, self.settings.clone(), self.ctx.reqid().into()); + let log_level = String::new(); + let subq = EventsSubQuery::from_parts(select, self.settings.clone(), self.ctx.reqid().into(), log_level); debug!("query for event blobs retrieval subq {subq:?}"); // TODO important TODO debug!("TODO fix magic inmem_bufcap"); @@ -922,12 +923,12 @@ impl Api1EventsBinaryHandler { return Err(Error::with_msg_no_trace("can not parse query")); } }; - let span = if qu.log_level() == "trace" { - debug!("enable trace for handler"); - tracing::span!(tracing::Level::TRACE, "log_span_trace") + let span = if false { + tracing::Span::none() + } else if qu.log_level() == "trace" { + tracing::span!(tracing::Level::INFO, "log_span_trace") } else if qu.log_level() == "debug" { - debug!("enable debug for handler"); - tracing::span!(tracing::Level::DEBUG, "log_span_debug") + tracing::span!(tracing::Level::INFO, "log_span_debug") } else { tracing::Span::none() }; diff --git a/crates/httpret/src/api4.rs b/crates/httpret/src/api4.rs index ac8d0ca..523da12 100644 --- a/crates/httpret/src/api4.rs +++ b/crates/httpret/src/api4.rs @@ -1,4 +1,5 @@ pub mod accounting; +pub mod backend; pub mod binned; pub mod databuffer_tools; pub mod docs; diff --git a/crates/httpret/src/api4/accounting.rs b/crates/httpret/src/api4/accounting.rs index 36c4f12..0c12850 100644 --- a/crates/httpret/src/api4/accounting.rs +++ b/crates/httpret/src/api4/accounting.rs @@ -19,10 +19,12 @@ use netpod::log::*; use netpod::req_uri_to_url; use netpod::FromUrl; use netpod::NodeConfigCached; +use netpod::Shape; use query::api4::AccountingIngestedBytesQuery; use query::api4::AccountingToplistQuery; use serde::Deserialize; use serde::Serialize; +use std::collections::BTreeMap; pub struct AccountingIngestedBytes {} @@ -87,7 +89,33 @@ impl AccountingIngestedBytes { #[derive(Debug, Serialize, Deserialize)] pub struct Toplist { - toplist: Vec<(String, u64, u64)>, + dim0: Vec<(String, u64, u64)>, + dim1: Vec<(String, u64, u64)>, + infos_count_total: usize, + infos_missing_count: usize, + top1_usage_len: usize, + scalar_count: usize, + wave_count: usize, + found: usize, + incomplete_count: usize, + mismatch_count: usize, +} + +impl Toplist { + fn new() -> Self { + Self { + dim0: Vec::new(), + dim1: Vec::new(), + infos_count_total: 0, + infos_missing_count: 0, + top1_usage_len: 0, + scalar_count: 0, + wave_count: 0, + found: 0, + incomplete_count: 0, + mismatch_count: 0, + } + } } pub struct AccountingToplistCounts {} @@ -135,6 +163,7 @@ impl AccountingToplistCounts { _ctx: &ReqCtx, ncc: &NodeConfigCached, ) -> Result { + let list_len_max = qu.limit() as usize; // TODO assumes that accounting data is in the LT keyspace let scyco = ncc .node_config @@ -145,22 +174,74 @@ impl AccountingToplistCounts { let pgconf = &ncc.node_config.cluster.database; let (pg, pgjh) = dbconn::create_connection(&pgconf).await?; let mut top1 = scyllaconn::accounting::toplist::read_ts(qu.ts().ns(), scy).await?; - top1.sort_by_bytes(); - let mut ret = Toplist { toplist: Vec::new() }; - let series_ids: Vec<_> = top1.usage().iter().take(qu.limit() as _).map(|x| x.0).collect(); - let infos = dbconn::channelinfo::info_for_series_ids(&series_ids, &pg) - .await - .map_err(Error::from_to_string)?; - let mut it = top1.usage().iter(); - for info in infos { - let h = it.next().ok_or_else(|| Error::with_msg_no_trace("logic error"))?; - if info.series != h.0 { - let e = Error::with_msg_no_trace(format!("mismatch {} != {}", info.series, h.0)); - warn!("{e}"); - return Err(e); + top1.sort_by_counts(); + let mut ret = Toplist::new(); + let top1_usage = top1.usage(); + ret.top1_usage_len = top1_usage.len(); + let usage_map_0: BTreeMap = top1_usage.iter().map(|x| (x.0, (x.1, x.2))).collect(); + let mut usage_it = usage_map_0.iter(); + loop { + let mut series_ids = Vec::new(); + let mut usages = Vec::new(); + while let Some(u) = usage_it.next() { + series_ids.push(*u.0); + usages.push(u.1.clone()); + if series_ids.len() >= 200 { + break; + } + } + if series_ids.len() == 0 { + break; + } + let infos = dbconn::channelinfo::info_for_series_ids(&series_ids, &pg) + .await + .map_err(Error::from_to_string)?; + for (_series, info_res) in &infos { + if let Some(info) = info_res { + match &info.shape { + Shape::Scalar => { + ret.scalar_count += 1; + } + Shape::Wave(_) => { + ret.wave_count += 1; + } + _ => {} + } + } + } + if usages.len() > infos.len() { + ret.incomplete_count += usages.len() - infos.len(); + } + if infos.len() > usages.len() { + ret.incomplete_count += infos.len() - usages.len(); + } + for ((series2, info_res), usage) in infos.into_iter().zip(usages.into_iter()) { + if let Some(info) = info_res { + if series2 != info.series { + ret.mismatch_count += 1; + } + ret.infos_count_total += 1; + // if info.name == "SINSB04-RMOD:PULSE-I-WF" { + // ret.found += 1; + // } + match &info.shape { + Shape::Scalar => { + ret.dim0.push((info.name, usage.0, usage.1)); + } + Shape::Wave(_) => { + ret.dim1.push((info.name, usage.0, usage.1)); + } + Shape::Image(_, _) => {} + } + } else { + ret.infos_missing_count += 1; + } } - ret.toplist.push((info.name, h.1, h.2)); } + ret.dim0.sort_by_cached_key(|x| u64::MAX - x.1); + ret.dim1.sort_by_cached_key(|x| u64::MAX - x.1); + ret.dim0.truncate(list_len_max); + ret.dim1.truncate(list_len_max); Ok(ret) } } diff --git a/crates/httpret/src/api4/backend.rs b/crates/httpret/src/api4/backend.rs new file mode 100644 index 0000000..ce9dbfe --- /dev/null +++ b/crates/httpret/src/api4/backend.rs @@ -0,0 +1,55 @@ +use crate::bodystream::response; +use crate::err::Error; +use crate::requests::accepts_json_or_all; +use http::Method; +use http::StatusCode; +use httpclient::body_empty; +use httpclient::body_string; +use httpclient::Requ; +use httpclient::StreamResponse; +use netpod::NodeConfigCached; +use netpod::ReqCtx; +use netpod::ServiceVersion; +use std::collections::BTreeMap; + +pub struct BackendListHandler {} + +impl BackendListHandler { + pub fn handler(req: &Requ) -> Option { + if req.uri().path() == "/api/4/backend/list" { + Some(Self {}) + } else { + None + } + } + + pub async fn handle( + &self, + req: Requ, + _ctx: &ReqCtx, + ncc: &NodeConfigCached, + _service_version: &ServiceVersion, + ) -> Result { + if req.method() == Method::GET { + if accepts_json_or_all(req.headers()) { + let mut list = Vec::new(); + if let Some(g) = &ncc.node_config.cluster.announce_backends { + for j in g { + let mut map = BTreeMap::new(); + map.insert("name", j.clone()); + list.push(map); + } + } + let res = serde_json::json!({ + "backends_available": list, + }); + let body = serde_json::to_string(&res)?; + Ok(response(StatusCode::OK).body(body_string(body))?) + } else { + Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?) + } + } else { + Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?) + } + } +} diff --git a/crates/httpret/src/api4/eventdata.rs b/crates/httpret/src/api4/eventdata.rs index f5d6bf1..ba6fde0 100644 --- a/crates/httpret/src/api4/eventdata.rs +++ b/crates/httpret/src/api4/eventdata.rs @@ -16,6 +16,7 @@ use httpclient::StreamResponse; use netpod::log::*; use netpod::NodeConfigCached; use std::sync::Arc; +use tracing::Instrument; #[derive(Debug, ThisError)] pub enum EventDataError { @@ -84,7 +85,19 @@ impl EventDataHandler { .await .map_err(|_| EventDataError::InternalError)?; let (evsubq,) = nodenet::conn::events_parse_input_query(frames).map_err(|_| EventDataError::QueryParse)?; + let logspan = if false { + tracing::Span::none() + } else if evsubq.log_level() == "trace" { + trace!("enable trace for handler"); + tracing::span!(tracing::Level::INFO, "log_span_trace") + } else if evsubq.log_level() == "debug" { + debug!("enable debug for handler"); + tracing::span!(tracing::Level::INFO, "log_span_debug") + } else { + tracing::Span::none() + }; let stream = nodenet::conn::create_response_bytes_stream(evsubq, shared_res.scyqueue.as_ref(), ncc) + .instrument(logspan) .await .map_err(|e| EventDataError::Error(Box::new(e)))?; let ret = response(StatusCode::OK) diff --git a/crates/httpret/src/api4/events.rs b/crates/httpret/src/api4/events.rs index 684c9d8..46c3c13 100644 --- a/crates/httpret/src/api4/events.rs +++ b/crates/httpret/src/api4/events.rs @@ -29,7 +29,7 @@ use netpod::NodeConfigCached; use netpod::ReqCtx; use nodenet::client::OpenBoxedBytesViaHttp; use query::api4::events::PlainEventsQuery; -use url::Url; +use tracing::Instrument; pub struct EventsHandler {} @@ -52,7 +52,26 @@ impl EventsHandler { if req.method() != Method::GET { return Ok(response(StatusCode::NOT_ACCEPTABLE).body(body_empty())?); } - match plain_events(req, ctx, &shared_res.pgqueue, ncc).await { + let self_name = "handle"; + let url = req_uri_to_url(req.uri())?; + let evq = + PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?; + debug!("{self_name} evq {evq:?}"); + let logspan = if false { + tracing::Span::none() + } else if evq.log_level() == "trace" { + trace!("enable trace for handler"); + tracing::span!(tracing::Level::INFO, "log_span_trace") + } else if evq.log_level() == "debug" { + debug!("enable debug for handler"); + tracing::span!(tracing::Level::INFO, "log_span_debug") + } else { + tracing::Span::none() + }; + match plain_events(req, evq, ctx, &shared_res.pgqueue, ncc) + .instrument(logspan) + .await + { Ok(ret) => Ok(ret), Err(e) => { error!("EventsHandler sees: {e}"); @@ -64,17 +83,17 @@ impl EventsHandler { async fn plain_events( req: Requ, + evq: PlainEventsQuery, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached, ) -> Result { - let url = req_uri_to_url(req.uri())?; if accepts_cbor_framed(req.headers()) { - Ok(plain_events_cbor_framed(url, req, ctx, pgqueue, ncc).await?) + Ok(plain_events_cbor_framed(req, evq, ctx, pgqueue, ncc).await?) } else if accepts_json_framed(req.headers()) { - Ok(plain_events_json_framed(url, req, ctx, pgqueue, ncc).await?) + Ok(plain_events_json_framed(req, evq, ctx, pgqueue, ncc).await?) } else if accepts_json_or_all(req.headers()) { - Ok(plain_events_json(url, req, ctx, pgqueue, ncc).await?) + Ok(plain_events_json(req, evq, ctx, pgqueue, ncc).await?) } else { let ret = response_err_msg(StatusCode::NOT_ACCEPTABLE, format!("unsupported accept {:?}", req))?; Ok(ret) @@ -82,17 +101,16 @@ async fn plain_events( } async fn plain_events_cbor_framed( - url: Url, req: Requ, + evq: PlainEventsQuery, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached, ) -> Result { - let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?; let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc) .await? .ok_or_else(|| Error::with_msg_no_trace("channel not found"))?; - info!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}"); + debug!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}"); let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()); let stream = streams::plaineventscbor::plain_events_cbor_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?; use future::ready; @@ -121,17 +139,16 @@ async fn plain_events_cbor_framed( } async fn plain_events_json_framed( - url: Url, req: Requ, + evq: PlainEventsQuery, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached, ) -> Result { - let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?; let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc) .await? .ok_or_else(|| Error::with_msg_no_trace("channel not found"))?; - info!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}"); + debug!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}"); let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()); let stream = streams::plaineventsjson::plain_events_json_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?; let stream = bytes_chunks_to_framed(stream); @@ -140,33 +157,26 @@ async fn plain_events_json_framed( } async fn plain_events_json( - url: Url, req: Requ, + evq: PlainEventsQuery, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached, ) -> Result { let self_name = "plain_events_json"; - info!("{self_name} req: {:?}", req); + debug!("{self_name} req: {:?}", req); let (_head, _body) = req.into_parts(); - let query = PlainEventsQuery::from_url(&url)?; - info!("{self_name} query {query:?}"); // TODO handle None case better and return 404 - let ch_conf = chconf_from_events_quorum(&query, ctx, pgqueue, ncc) + let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc) .await .map_err(Error::from)? .ok_or_else(|| Error::with_msg_no_trace("channel not found"))?; - info!("{self_name} chconf_from_events_quorum: {ch_conf:?}"); + debug!("{self_name} chconf_from_events_quorum: {ch_conf:?}"); let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone()); - let item = streams::plaineventsjson::plain_events_json( - &query, - ch_conf, - ctx, - &ncc.node_config.cluster, - Box::pin(open_bytes), - ) - .await; - info!("{self_name} returned {}", item.is_ok()); + let item = + streams::plaineventsjson::plain_events_json(&evq, ch_conf, ctx, &ncc.node_config.cluster, Box::pin(open_bytes)) + .await; + debug!("{self_name} returned {}", item.is_ok()); let item = match item { Ok(item) => item, Err(e) => { @@ -175,7 +185,7 @@ async fn plain_events_json( } }; let ret = response(StatusCode::OK).body(ToJsonBody::from(&item).into_body())?; - info!("{self_name} response created"); + debug!("{self_name} response created"); Ok(ret) } diff --git a/crates/httpret/src/httpret.rs b/crates/httpret/src/httpret.rs index 2ed4581..7d7104c 100644 --- a/crates/httpret/src/httpret.rs +++ b/crates/httpret/src/httpret.rs @@ -127,10 +127,12 @@ pub async fn host(ncc: NodeConfigCached, service_version: ServiceVersion) -> Res ncc.node_config.cluster.scylla_mt(), ncc.node_config.cluster.scylla_lt(), ) { - let (scyqueue, scylla_worker) = ScyllaWorker::new(st, mt, lt).await.map_err(|e| { - error!("{e}"); - RetrievalError::TextError(e.to_string()) - })?; + let (scyqueue, scylla_worker) = ScyllaWorker::new(st.clone(), mt.clone(), lt.clone()) + .await + .map_err(|e| { + error!("{e}"); + RetrievalError::TextError(e.to_string()) + })?; let scylla_worker_jh = taskrun::spawn(scylla_worker.work()); Some(scyqueue) } else { @@ -342,6 +344,8 @@ async fn http_service_inner( Ok(h.handle(req, ctx, &node_config, shared_res) .await .map_err(|e| Error::with_msg_no_trace(e.to_string()))?) + } else if let Some(h) = api4::backend::BackendListHandler::handler(&req) { + Ok(h.handle(req, ctx, &node_config, service_version).await?) } else if let Some(h) = api4::status::StatusNodesRecursive::handler(&req) { Ok(h.handle(req, ctx, &node_config, service_version).await?) } else if let Some(h) = StatusBoardAllHandler::handler(&req) { diff --git a/crates/httpret/src/proxy.rs b/crates/httpret/src/proxy.rs index c9aec73..c95f3f3 100644 --- a/crates/httpret/src/proxy.rs +++ b/crates/httpret/src/proxy.rs @@ -44,6 +44,7 @@ use netpod::ChannelSearchSingleResult; use netpod::FromUrl; use netpod::HasBackend; use netpod::HasTimeout; +use netpod::MapQuery; use netpod::ProxyConfig; use netpod::ReqCtx; use netpod::ServiceVersion; @@ -196,6 +197,8 @@ async fn proxy_http_service_inner( h.handle(req, ctx, &proxy_config).await } else if let Some(h) = api4::events::EventsHandler::handler(&req) { h.handle(req, ctx, &proxy_config).await + } else if path == "/api/4/accounting/toplist/counts" { + Ok(proxy_backend_query::(req, ctx, proxy_config).await?) } else if path == "/api/4/status/connection/events" { Ok(proxy_backend_query::(req, ctx, proxy_config).await?) } else if path == "/api/4/status/channel/events" { diff --git a/crates/httpret/src/proxy/api4/backend.rs b/crates/httpret/src/proxy/api4/backend.rs index 85abaa2..9ef2101 100644 --- a/crates/httpret/src/proxy/api4/backend.rs +++ b/crates/httpret/src/proxy/api4/backend.rs @@ -32,6 +32,9 @@ impl BackendListHandler { { "name": "sf-imagebuffer", }, + { + "name": "sf-archiver", + }, ] }); let body = serde_json::to_string(&res)?; diff --git a/crates/items_0/src/items_0.rs b/crates/items_0/src/items_0.rs index b0a1c7b..a10788a 100644 --- a/crates/items_0/src/items_0.rs +++ b/crates/items_0/src/items_0.rs @@ -128,7 +128,7 @@ pub trait Events: fn as_time_binnable_ref(&self) -> &dyn TimeBinnable; fn as_time_binnable_mut(&mut self) -> &mut dyn TimeBinnable; fn verify(&self) -> bool; - fn output_info(&self); + fn output_info(&self) -> String; fn as_collectable_mut(&mut self) -> &mut dyn Collectable; fn as_collectable_with_default_ref(&self) -> &dyn Collectable; fn as_collectable_with_default_mut(&mut self) -> &mut dyn Collectable; @@ -190,7 +190,7 @@ impl Events for Box { Events::verify(self.as_ref()) } - fn output_info(&self) { + fn output_info(&self) -> String { Events::output_info(self.as_ref()) } diff --git a/crates/items_0/src/scalar_ops.rs b/crates/items_0/src/scalar_ops.rs index 433d56d..3cc4a04 100644 --- a/crates/items_0/src/scalar_ops.rs +++ b/crates/items_0/src/scalar_ops.rs @@ -1,5 +1,7 @@ use crate::container::ByteEstimate; use crate::subfr::SubFrId; +use netpod::EnumVariant; +use netpod::StringFix; use serde::Serialize; use std::fmt; use std::ops; @@ -247,3 +249,49 @@ impl_scalar_ops!( "string", 16 ); + +impl ByteEstimate for EnumVariant { + fn byte_estimate(&self) -> u64 { + 12 + } +} + +impl AsPrimF32 for EnumVariant { + fn as_prim_f32_b(&self) -> f32 { + 0. + } +} + +impl ScalarOps for EnumVariant { + fn scalar_type_name() -> &'static str { + "enumvariant" + } + + fn zero_b() -> Self { + EnumVariant::empty() + } + + fn equal_slack(&self, rhs: &Self) -> bool { + self == rhs + } + + fn add(&mut self, _rhs: &Self) { + // undefined so far + } + + fn div(&mut self, _n: usize) { + // undefined so far + } + + fn find_vec_min(a: &Vec) -> Option { + todo!() + } + + fn find_vec_max(a: &Vec) -> Option { + todo!() + } + + fn avg_vec(a: &Vec) -> Option { + todo!() + } +} diff --git a/crates/items_0/src/subfr.rs b/crates/items_0/src/subfr.rs index e339fe6..25ac5b9 100644 --- a/crates/items_0/src/subfr.rs +++ b/crates/items_0/src/subfr.rs @@ -1,3 +1,5 @@ +use netpod::EnumVariant; + pub trait SubFrId { const SUB: u32; } @@ -49,3 +51,7 @@ impl SubFrId for bool { impl SubFrId for String { const SUB: u32 = 0x0e; } + +impl SubFrId for EnumVariant { + const SUB: u32 = 0x0f; +} diff --git a/crates/items_2/src/channelevents.rs b/crates/items_2/src/channelevents.rs index ea3f249..f918917 100644 --- a/crates/items_2/src/channelevents.rs +++ b/crates/items_2/src/channelevents.rs @@ -832,7 +832,7 @@ impl Events for ChannelEvents { todo!() } - fn output_info(&self) { + fn output_info(&self) -> String { todo!() } @@ -1220,11 +1220,15 @@ impl Collector for ChannelEventsCollector { ) -> Result, err::Error> { match self.coll.as_mut() { Some(coll) => { - coll.set_continue_at_here(); + if self.needs_continue_at { + debug!("ChannelEventsCollector set_continue_at_here"); + coll.set_continue_at_here(); + } if self.range_complete { coll.set_range_complete(); } if self.timed_out { + debug!("ChannelEventsCollector set_timed_out"); coll.set_timed_out(); } let res = coll.result(range, binrange)?; diff --git a/crates/items_2/src/empty.rs b/crates/items_2/src/empty.rs index a583336..26c6edf 100644 --- a/crates/items_2/src/empty.rs +++ b/crates/items_2/src/empty.rs @@ -4,6 +4,7 @@ use crate::Error; use items_0::Empty; use items_0::Events; use netpod::log::*; +use netpod::EnumVariant; use netpod::ScalarType; use netpod::Shape; @@ -25,6 +26,7 @@ pub fn empty_events_dyn_ev(scalar_type: &ScalarType, shape: &Shape) -> Result Box::new(K::::empty()), BOOL => Box::new(K::::empty()), STRING => Box::new(K::::empty()), + Enum => Box::new(K::::empty()), ChannelStatus => Box::new(K::::empty()), } } @@ -44,6 +46,7 @@ pub fn empty_events_dyn_ev(scalar_type: &ScalarType, shape: &Shape) -> Result Box::new(K::::empty()), BOOL => Box::new(K::::empty()), STRING => Box::new(K::::empty()), + Enum => Box::new(K::::empty()), ChannelStatus => Box::new(K::::empty()), } } diff --git a/crates/items_2/src/eventsdim0.rs b/crates/items_2/src/eventsdim0.rs index 9eb32db..dba7ea3 100644 --- a/crates/items_2/src/eventsdim0.rs +++ b/crates/items_2/src/eventsdim0.rs @@ -812,38 +812,25 @@ impl Events for EventsDim0 { fn verify(&self) -> bool { let mut good = true; - let mut ts_max = 0; - for ts in &self.tss { - let ts = *ts; - if ts < ts_max { + let n = self.tss.len(); + for (&ts1, &ts2) in self.tss.iter().zip(self.tss.range(n.min(1)..n)) { + if ts1 > ts2 { good = false; - error!("unordered event data ts {} ts_max {}", ts, ts_max); + error!("unordered event data ts1 {} ts2 {}", ts1, ts2); + break; } - ts_max = ts_max.max(ts); } good } - fn output_info(&self) { - if false { - info!("output_info len {}", self.tss.len()); - if self.tss.len() == 1 { - info!( - " only: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.values[0] - ); - } else if self.tss.len() > 1 { - info!( - " first: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.values[0] - ); - let n = self.tss.len() - 1; - info!( - " last: ts {} pulse {} value {:?}", - self.tss[n], self.pulses[n], self.values[n] - ); - } - } + fn output_info(&self) -> String { + let n2 = self.tss.len().max(1) - 1; + format!( + "EventsDim0OutputInfo {{ len {}, ts_min {}, ts_max {} }}", + self.tss.len(), + self.tss.get(0).map_or(-1i64, |&x| x as i64), + self.tss.get(n2).map_or(-1i64, |&x| x as i64), + ) } fn as_collectable_mut(&mut self) -> &mut dyn Collectable { diff --git a/crates/items_2/src/eventsdim1.rs b/crates/items_2/src/eventsdim1.rs index 8671b31..33250b2 100644 --- a/crates/items_2/src/eventsdim1.rs +++ b/crates/items_2/src/eventsdim1.rs @@ -777,26 +777,14 @@ impl Events for EventsDim1 { good } - fn output_info(&self) { - if false { - info!("output_info len {}", self.tss.len()); - if self.tss.len() == 1 { - info!( - " only: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.values[0] - ); - } else if self.tss.len() > 1 { - info!( - " first: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.values[0] - ); - let n = self.tss.len() - 1; - info!( - " last: ts {} pulse {} value {:?}", - self.tss[n], self.pulses[n], self.values[n] - ); - } - } + fn output_info(&self) -> String { + let n2 = self.tss.len().max(1) - 1; + format!( + "EventsDim1OutputInfo {{ len {}, ts_min {}, ts_max {} }}", + self.tss.len(), + self.tss.get(0).map_or(-1i64, |&x| x as i64), + self.tss.get(n2).map_or(-1i64, |&x| x as i64), + ) } fn as_collectable_mut(&mut self) -> &mut dyn Collectable { diff --git a/crates/items_2/src/eventsxbindim0.rs b/crates/items_2/src/eventsxbindim0.rs index 4b16541..62102d9 100644 --- a/crates/items_2/src/eventsxbindim0.rs +++ b/crates/items_2/src/eventsxbindim0.rs @@ -217,26 +217,14 @@ impl Events for EventsXbinDim0 { good } - fn output_info(&self) { - if false { - info!("output_info len {}", self.tss.len()); - if self.tss.len() == 1 { - info!( - " only: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.avgs[0] - ); - } else if self.tss.len() > 1 { - info!( - " first: ts {} pulse {} value {:?}", - self.tss[0], self.pulses[0], self.avgs[0] - ); - let n = self.tss.len() - 1; - info!( - " last: ts {} pulse {} value {:?}", - self.tss[n], self.pulses[n], self.avgs[n] - ); - } - } + fn output_info(&self) -> String { + let n2 = self.tss.len().max(1) - 1; + format!( + "EventsXbinDim0OutputInfo {{ len {}, ts_min {}, ts_max {} }}", + self.tss.len(), + self.tss.get(0).map_or(-1i64, |&x| x as i64), + self.tss.get(n2).map_or(-1i64, |&x| x as i64), + ) } fn as_collectable_mut(&mut self) -> &mut dyn Collectable { diff --git a/crates/netpod/src/netpod.rs b/crates/netpod/src/netpod.rs index 72af0cc..9abcfa9 100644 --- a/crates/netpod/src/netpod.rs +++ b/crates/netpod/src/netpod.rs @@ -121,7 +121,7 @@ pub struct BodyStream { pub inner: Box> + Send + Unpin>, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum SeriesKind { ChannelStatus, ChannelData, @@ -160,6 +160,7 @@ pub enum ScalarType { F64, BOOL, STRING, + Enum, ChannelStatus, } @@ -194,6 +195,7 @@ impl Serialize for ScalarType { F64 => ser.serialize_str("f64"), BOOL => ser.serialize_str("bool"), STRING => ser.serialize_str("string"), + Enum => ser.serialize_str("enum"), ChannelStatus => ser.serialize_str("ChannelStatus"), } } @@ -223,6 +225,7 @@ impl<'de> serde::de::Visitor<'de> for ScalarTypeVis { "f64" => ScalarType::F64, "bool" => ScalarType::BOOL, "string" => ScalarType::STRING, + "enum" => ScalarType::Enum, "channelstatus" => ScalarType::ChannelStatus, k => return Err(E::custom(format!("can not understand variant {k:?}"))), }; @@ -261,7 +264,7 @@ impl ScalarType { 12 => F64, 13 => STRING, 14 => ChannelStatus, - //13 => return Err(Error::with_msg(format!("STRING not supported"))), + 15 => Enum, 6 => return Err(Error::with_msg(format!("CHARACTER not supported"))), _ => return Err(Error::with_msg(format!("unknown dtype code: {:?}", ix))), }; @@ -283,6 +286,7 @@ impl ScalarType { F64 => "f64", BOOL => "bool", STRING => "string", + Enum => "enum", ChannelStatus => "ChannelStatus", } } @@ -302,6 +306,7 @@ impl ScalarType { "f64" => F64, "bool" => BOOL, "string" => STRING, + "enum" => Enum, "ChannelStatus" => ChannelStatus, _ => { return Err(Error::with_msg_no_trace(format!( @@ -328,6 +333,7 @@ impl ScalarType { F64 => "float64", BOOL => "bool", STRING => "string", + Enum => "enum", ChannelStatus => "ChannelStatus", } } @@ -347,8 +353,9 @@ impl ScalarType { "double" => F64, "float32" => F32, "float64" => F64, - "string" => STRING, "bool" => BOOL, + "string" => STRING, + "enum" => Enum, "ChannelStatus" => ChannelStatus, _ => { return Err(Error::with_msg_no_trace(format!( @@ -366,7 +373,7 @@ impl ScalarType { 0 => STRING, 1 => I16, 2 => F32, - 3 => I16, + 3 => Enum, 4 => I8, 5 => I32, 6 => F64, @@ -389,6 +396,7 @@ impl ScalarType { F32 => 2, F64 => 6, STRING => 0, + Enum => 3, _ => return Err(Error::with_msg_no_trace(format!("can not represent {self:?} as CA id"))), }; Ok(ret) @@ -420,6 +428,7 @@ impl ScalarType { Self::from_dtype_index(k as u8) } + // TODO this is useless for strings and enums. pub fn bytes(&self) -> u8 { use ScalarType::*; match self { @@ -435,6 +444,7 @@ impl ScalarType { F64 => 8, BOOL => 1, STRING => 1, + Enum => 2, ChannelStatus => 4, } } @@ -455,6 +465,7 @@ impl ScalarType { BOOL => 0, STRING => 13, ChannelStatus => 14, + Enum => 15, } } @@ -468,6 +479,77 @@ impl ScalarType { } } +#[derive(Debug, Clone, PartialOrd, PartialEq)] +pub struct StringFix { + data: [char; N], +} + +impl StringFix { + pub fn new() -> Self { + Self { + data: [char::REPLACEMENT_CHARACTER; N], + } + } +} + +mod string_fix_impl_serde { + use crate::StringFix; + use serde::de::Visitor; + use serde::Deserialize; + use serde::Serialize; + use std::fmt; + + impl Serialize for StringFix { + fn serialize(&self, ser: S) -> Result + where + S: serde::Serializer, + { + ser.serialize_unit() + } + } + + impl<'de, const N: usize> Deserialize<'de> for StringFix { + fn deserialize(de: D) -> Result + where + D: serde::Deserializer<'de>, + { + de.deserialize_unit(Vis::) + } + } + + struct Vis; + + impl<'de, const N: usize> Visitor<'de> for Vis { + type Value = StringFix; + + fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "deserialize enum error") + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(Self::Value::new()) + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialOrd, PartialEq)] +pub struct EnumVariant { + ix: u16, + name: StringFix<26>, +} + +impl EnumVariant { + pub fn empty() -> Self { + Self { + ix: u16::MAX, + name: StringFix::new(), + } + } +} + impl AppendToUrl for ScalarType { fn append_to_url(&self, url: &mut Url) { let mut g = url.query_pairs_mut(); @@ -650,6 +732,7 @@ pub struct Cluster { #[serde(rename = "scylla_lt")] scylla_lt: Option, cache_scylla: Option, + pub announce_backends: Option>, } impl Cluster { @@ -692,6 +775,7 @@ impl Cluster { scylla_mt: None, scylla_lt: None, cache_scylla: None, + announce_backends: None, } } } @@ -2879,6 +2963,50 @@ pub trait AppendToUrl { fn append_to_url(&self, url: &mut Url); } +pub type MapQuery = BTreeMap; + +impl AppendToUrl for MapQuery { + fn append_to_url(&self, url: &mut Url) { + let mut g = url.query_pairs_mut(); + for (k, v) in self { + g.append_pair(k, v); + } + } +} + +impl FromUrl for MapQuery { + fn from_url(url: &Url) -> Result { + let pairs = get_url_query_pairs(url); + Self::from_pairs(&pairs) + } + + fn from_pairs(pairs: &BTreeMap) -> Result { + Ok(pairs.clone()) + } +} + +impl HasBackend for MapQuery { + fn backend(&self) -> &str { + self.get("backend").map_or("NOBACKEND", AsRef::as_ref) + } +} + +impl HasTimeout for MapQuery { + fn timeout(&self) -> Duration { + let x: Option = if let Some(v) = self.get("timeout") { + v.parse::().ok() + } else { + None + }; + let x = x.unwrap_or(5000); + Duration::from_millis(x as _) + } + + fn set_timeout(&mut self, timeout: Duration) { + self.insert("timeout".into(), format!("{:.0}", 1e3 * timeout.as_secs_f32())); + } +} + pub fn get_url_query_pairs(url: &Url) -> BTreeMap { BTreeMap::from_iter(url.query_pairs().map(|(j, k)| (j.to_string(), k.to_string()))) } @@ -3291,6 +3419,7 @@ pub fn test_cluster() -> Cluster { run_map_pulse_task: false, is_central_storage: false, file_io_buffer_size: Default::default(), + announce_backends: None, } } @@ -3328,6 +3457,7 @@ pub fn sls_test_cluster() -> Cluster { run_map_pulse_task: false, is_central_storage: false, file_io_buffer_size: Default::default(), + announce_backends: None, } } @@ -3365,6 +3495,7 @@ pub fn archapp_test_cluster() -> Cluster { run_map_pulse_task: false, is_central_storage: false, file_io_buffer_size: Default::default(), + announce_backends: None, } } diff --git a/crates/nodenet/src/conn/test.rs b/crates/nodenet/src/conn/test.rs index a0e39c9..70b897f 100644 --- a/crates/nodenet/src/conn/test.rs +++ b/crates/nodenet/src/conn/test.rs @@ -84,7 +84,8 @@ fn raw_data_00() { ); let select = EventsSubQuerySelect::new(fetch_info.into(), range.into(), TransformQuery::default_events()); let settings = EventsSubQuerySettings::default(); - let qu = EventsSubQuery::from_parts(select, settings, "dummy".into()); + let log_level = String::new(); + let qu = EventsSubQuery::from_parts(select, settings, "dummy".into(), log_level); let frame1 = Frame1Parts::new(qu.clone()); let query = EventQueryJsonStringFrame(serde_json::to_string(&frame1).unwrap()); let frame = sitem_data(query).make_frame()?; diff --git a/crates/parse/src/api1_parse.rs b/crates/parse/src/api1_parse.rs index fa09d81..9e4b78f 100644 --- a/crates/parse/src/api1_parse.rs +++ b/crates/parse/src/api1_parse.rs @@ -110,6 +110,8 @@ impl From<&ScalarType> for Api1ScalarType { A::F64 => B::F64, A::BOOL => B::BOOL, A::STRING => B::STRING, + // TODO treat enum as number only + A::Enum => B::U16, A::ChannelStatus => todo!("ChannelStatus not in Api1ScalarType"), } } diff --git a/crates/query/src/api4/binned.rs b/crates/query/src/api4/binned.rs index 939f6db..cc2550a 100644 --- a/crates/query/src/api4/binned.rs +++ b/crates/query/src/api4/binned.rs @@ -148,6 +148,11 @@ impl BinnedQuery { None => None, } } + + pub fn log_level(&self) -> &str { + // TODO take from query + "" + } } impl HasBackend for BinnedQuery { diff --git a/crates/query/src/api4/events.rs b/crates/query/src/api4/events.rs index 01b7cc2..c9e5eab 100644 --- a/crates/query/src/api4/events.rs +++ b/crates/query/src/api4/events.rs @@ -54,6 +54,8 @@ pub struct PlainEventsQuery { merger_out_len_max: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] create_errors: Vec, + #[serde(default)] + log_level: String, } impl PlainEventsQuery { @@ -78,6 +80,7 @@ impl PlainEventsQuery { test_do_wasm: None, merger_out_len_max: None, create_errors: Vec::new(), + log_level: String::new(), } } @@ -199,6 +202,10 @@ impl PlainEventsQuery { self.range() ) } + + pub fn log_level(&self) -> &str { + &self.log_level + } } impl HasBackend for PlainEventsQuery { @@ -275,6 +282,7 @@ impl FromUrl for PlainEventsQuery { .get("create_errors") .map(|x| x.split(",").map(|x| x.to_string()).collect()) .unwrap_or(Vec::new()), + log_level: pairs.get("log_level").map_or(String::new(), String::from), }; Ok(ret) } @@ -331,6 +339,9 @@ impl AppendToUrl for PlainEventsQuery { if self.create_errors.len() != 0 { g.append_pair("create_errors", &self.create_errors.join(",")); } + if self.log_level.len() != 0 { + g.append_pair("log_level", &self.log_level); + } } } @@ -447,15 +458,22 @@ pub struct EventsSubQuery { settings: EventsSubQuerySettings, ty: String, reqid: String, + log_level: String, } impl EventsSubQuery { - pub fn from_parts(select: EventsSubQuerySelect, settings: EventsSubQuerySettings, reqid: String) -> Self { + pub fn from_parts( + select: EventsSubQuerySelect, + settings: EventsSubQuerySettings, + reqid: String, + log_level: String, + ) -> Self { Self { select, settings, ty: "EventsSubQuery".into(), reqid, + log_level, } } @@ -529,6 +547,10 @@ impl EventsSubQuery { pub fn wasm1(&self) -> Option<&str> { self.select.wasm1() } + + pub fn log_level(&self) -> &str { + &self.log_level + } } #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/scyllaconn/src/accounting/toplist.rs b/crates/scyllaconn/src/accounting/toplist.rs index 2842636..de1d94b 100644 --- a/crates/scyllaconn/src/accounting/toplist.rs +++ b/crates/scyllaconn/src/accounting/toplist.rs @@ -43,14 +43,14 @@ pub async fn read_ts(ts: u64, scy: Arc) -> Result let snap = EMIT_ACCOUNTING_SNAP.ms() / 1000; info!("ts {ts} snap {snap:?}"); let ts = ts / timeunits::SEC / snap * snap; - let cql = concat!("select series, count, bytes from lt_account_00 where part = ? and ts = ?"); - let qu = prep(cql, scy.clone()).await?; - let ret = read_ts_inner(ts, qu, scy).await?; + let ret = read_ts_inner(ts, scy).await?; Ok(ret) } -async fn read_ts_inner(ts: u64, qu: PreparedStatement, scy: Arc) -> Result { +async fn read_ts_inner(ts: u64, scy: Arc) -> Result { type RowType = (i64, i64, i64); + let cql = concat!("select series, count, bytes from lt_account_00 where part = ? and ts = ?"); + let qu = prep(cql, scy.clone()).await?; let mut ret = UsageData::new(ts); for part in 0..255_u32 { let mut res = scy diff --git a/crates/scyllaconn/src/events.rs b/crates/scyllaconn/src/events.rs index d49b2cc..3e0f7dd 100644 --- a/crates/scyllaconn/src/events.rs +++ b/crates/scyllaconn/src/events.rs @@ -224,6 +224,36 @@ macro_rules! impl_scaty_array { }; } +impl ValTy for Vec { + type ScaTy = String; + type ScyTy = Vec; + type Container = EventsDim1; + + fn from_scyty(inp: Self::ScyTy) -> Self { + inp + } + + fn from_valueblob(inp: Vec) -> Self { + todo!() + } + + fn table_name() -> &'static str { + "st_events_array_enum" + } + + fn default() -> Self { + Vec::new() + } + + fn is_valueblob() -> bool { + false + } + + fn st_name() -> &'static str { + "enum" + } +} + impl_scaty_scalar!(u8, i8, "u8", "st_events_scalar_u8"); impl_scaty_scalar!(u16, i16, "u16", "st_events_scalar_u16"); impl_scaty_scalar!(u32, i32, "u32", "st_events_scalar_u32"); @@ -248,7 +278,6 @@ impl_scaty_array!(Vec, i64, Vec, "i64", "st_events_array_i64"); impl_scaty_array!(Vec, f32, Vec, "f32", "st_events_array_f32"); impl_scaty_array!(Vec, f64, Vec, "f64", "st_events_array_f64"); impl_scaty_array!(Vec, bool, Vec, "bool", "st_events_array_bool"); -// impl_scaty_array!(Vec, String, Vec, "string", "st_events_array_string"); struct ReadNextValuesOpts { series: u64, @@ -307,14 +336,19 @@ where ts_lsp_max, table_name, ); + let dir = "fwd"; let qu_name = if opts.with_values { if ST::is_valueblob() { - format!("array_{}_valueblobs_fwd", ST::st_name()) + format!("array_{}_valueblobs_{}", ST::st_name(), dir) } else { - format!("array_{}_values_fwd", ST::st_name()) + format!("scalar_{}_values_{}", ST::st_name(), dir) } } else { - format!("array_{}_timestamps_fwd", ST::st_name()) + if ST::is_valueblob() { + format!("array_{}_timestamps_{}", ST::st_name(), dir) + } else { + format!("scalar_{}_timestamps_{}", ST::st_name(), dir) + } }; let qu = stmts.read_value_queries.get(&qu_name).ok_or_else(|| { let e = Error::with_msg_no_trace(format!("can not find query name {}", qu_name)); @@ -343,14 +377,19 @@ where DtNano::from_ns(0) }; trace!("BCK ts_msp {} ts_lsp_max {} {}", ts_msp, ts_lsp_max, table_name,); + let dir = "bck"; let qu_name = if opts.with_values { if ST::is_valueblob() { - format!("array_{}_valueblobs_bck", ST::st_name()) + format!("array_{}_valueblobs_{}", ST::st_name(), dir) } else { - format!("array_{}_values_bck", ST::st_name()) + format!("scalar_{}_values_{}", ST::st_name(), dir) } } else { - format!("array_{}_timestamps_bck", ST::st_name()) + if ST::is_valueblob() { + format!("array_{}_timestamps_{}", ST::st_name(), dir) + } else { + format!("scalar_{}_timestamps_{}", ST::st_name(), dir) + } }; let qu = stmts.read_value_queries.get(&qu_name).ok_or_else(|| { let e = Error::with_msg_no_trace(format!("can not find query name {}", qu_name)); @@ -512,6 +551,7 @@ impl ReadValues { ScalarType::F64 => read_next_values::(opts).await, ScalarType::BOOL => read_next_values::(opts).await, ScalarType::STRING => read_next_values::(opts).await, + ScalarType::Enum => read_next_values::(opts).await, ScalarType::ChannelStatus => { warn!("read scalar channel status not yet supported"); err::todoval() @@ -533,6 +573,7 @@ impl ReadValues { warn!("read array string not yet supported"); err::todoval() } + ScalarType::Enum => read_next_values::>(opts).await, ScalarType::ChannelStatus => { warn!("read array channel status not yet supported"); err::todoval() @@ -572,6 +613,7 @@ pub struct EventsStreamScylla { found_one_after: bool, with_values: bool, outqueue: VecDeque>, + ts_seen_max: u64, } impl EventsStreamScylla { @@ -600,6 +642,7 @@ impl EventsStreamScylla { found_one_after: false, with_values, outqueue: VecDeque::new(), + ts_seen_max: 0, } } @@ -618,7 +661,7 @@ impl EventsStreamScylla { trace!("ts_msp_bck {:?}", self.ts_msp_bck); trace!("ts_msp_fwd {:?}", self.ts_msp_fwd); if let Some(msp) = self.ts_msp_bck.pop_back() { - trace!("Try ReadBack1"); + trace!("start ReadBack1 msp {}", msp); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -631,7 +674,7 @@ impl EventsStreamScylla { ); self.state = FrState::ReadBack1(st); } else if self.ts_msp_fwd.len() > 0 { - trace!("Go straight for forward read"); + trace!("begin immediately with forward read"); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -653,6 +696,7 @@ impl EventsStreamScylla { if item.len() > 0 { self.outqueue.push_back(item); if self.ts_msp_fwd.len() > 0 { + trace!("start forward read after back1"); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -669,7 +713,7 @@ impl EventsStreamScylla { } } else { if let Some(msp) = self.ts_msp_bck.pop_back() { - trace!("Try ReadBack2"); + trace!("start ReadBack2 msp {}", msp); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -682,7 +726,7 @@ impl EventsStreamScylla { ); self.state = FrState::ReadBack2(st); } else if self.ts_msp_fwd.len() > 0 { - trace!("No 2nd back MSP, go for forward read"); + trace!("no 2nd back MSP, go for forward read"); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -695,18 +739,19 @@ impl EventsStreamScylla { ); self.state = FrState::ReadValues(st); } else { - trace!("No 2nd back MSP, but also nothing to go forward"); + trace!("no 2nd back msp, but also nothing to go forward"); self.state = FrState::DataDone; } } } fn back_2_done(&mut self, item: Box) { - trace!("back_1_done item len {}", item.len()); + trace!("back_2_done item len {}", item.len()); if item.len() > 0 { self.outqueue.push_back(item); } if self.ts_msp_fwd.len() > 0 { + trace!("start forward read after back2"); let st = ReadValues::new( self.series, self.scalar_type.clone(), @@ -719,6 +764,7 @@ impl EventsStreamScylla { ); self.state = FrState::ReadValues(st); } else { + trace!("nothing to forward read after back 2"); self.state = FrState::DataDone; } } @@ -745,7 +791,19 @@ impl Stream for EventsStreamScylla { loop { if let Some(item) = self.outqueue.pop_front() { item.verify(); - item.output_info(); + if let Some(item_min) = item.ts_min() { + if item_min < self.ts_seen_max { + debug!("ordering error A {} {}", item_min, self.ts_seen_max); + } + } + if let Some(item_max) = item.ts_max() { + if item_max < self.ts_seen_max { + debug!("ordering error B {} {}", item_max, self.ts_seen_max); + } else { + self.ts_seen_max = item_max; + } + } + debug!("deliver item {}", item.output_info()); break Ready(Some(Ok(ChannelEvents::Events(item)))); } break match self.state { diff --git a/crates/scyllaconn/src/worker.rs b/crates/scyllaconn/src/worker.rs index 52fe372..a934e80 100644 --- a/crates/scyllaconn/src/worker.rs +++ b/crates/scyllaconn/src/worker.rs @@ -100,28 +100,34 @@ impl ScyllaQueue { #[derive(Debug)] pub struct ScyllaWorker { rx: Receiver, - scy: Arc, - stmts_st: Arc, + scyconf_st: ScyllaConfig, + scyconf_mt: ScyllaConfig, + scyconf_lt: ScyllaConfig, } impl ScyllaWorker { pub async fn new( - scyconf_st: &ScyllaConfig, - scyconf_mt: &ScyllaConfig, - scyconf_lt: &ScyllaConfig, + scyconf_st: ScyllaConfig, + scyconf_mt: ScyllaConfig, + scyconf_lt: ScyllaConfig, ) -> Result<(ScyllaQueue, Self), Error> { let (tx, rx) = async_channel::bounded(64); - let scy = create_scy_session_no_ks(scyconf_st).await?; - let scy = Arc::new(scy); - let rtpre = format!("{}.st_", scyconf_st.keyspace); - let stmts_st = StmtsEventsRt::new(&rtpre, &scy).await?; - let stmts_st = Arc::new(stmts_st); let queue = ScyllaQueue { tx }; - let worker = Self { rx, scy, stmts_st }; + let worker = Self { + rx, + scyconf_st, + scyconf_mt, + scyconf_lt, + }; Ok((queue, worker)) } pub async fn work(self) -> Result<(), Error> { + let scy = create_scy_session_no_ks(&self.scyconf_st).await?; + let scy = Arc::new(scy); + let rtpre = format!("{}.st_", self.scyconf_st.keyspace); + let stmts_st = StmtsEventsRt::new(&rtpre, &scy).await?; + let stmts_st = Arc::new(stmts_st); loop { let x = self.rx.recv().await; let job = match x { @@ -133,13 +139,13 @@ impl ScyllaWorker { }; match job { Job::FindTsMsp(series, range, tx) => { - let res = crate::events::find_ts_msp_worker(series, range, &self.stmts_st, &self.scy).await; + let res = crate::events::find_ts_msp_worker(series, range, &stmts_st, &scy).await; if tx.send(res.map_err(Into::into)).await.is_err() { // TODO count for stats } } Job::ReadNextValues(job) => { - let fut = (job.futgen)(self.scy.clone(), self.stmts_st.clone()); + let fut = (job.futgen)(scy.clone(), stmts_st.clone()); let res = fut.await; if job.tx.send(res.map_err(Into::into)).await.is_err() { // TODO count for stats diff --git a/crates/streams/src/collect.rs b/crates/streams/src/collect.rs index 345eba3..857248c 100644 --- a/crates/streams/src/collect.rs +++ b/crates/streams/src/collect.rs @@ -166,6 +166,7 @@ impl Future for Collect { break if self.done_input { if self.timeout { if let Some(coll) = self.collector.as_mut() { + info!("Collect call set_timed_out"); coll.set_timed_out(); } else { warn!("collect timeout but no collector yet"); @@ -199,7 +200,7 @@ impl Future for Collect { continue; } Err(e) => { - error!("{e}"); + error!("Collect {e}"); Ready(Err(e)) } }, @@ -241,6 +242,7 @@ where warn!("collect timeout"); timed_out = true; if let Some(coll) = collector.as_mut() { + info!("collect_in_span call set_timed_out"); coll.set_timed_out(); } else { warn!("collect timeout but no collector yet"); @@ -269,6 +271,7 @@ where coll.ingest(&mut item); if coll.len() as u64 >= events_max { warn!("span reached events_max {}", events_max); + info!("collect_in_span call set_continue_at_here"); coll.set_continue_at_here(); break; } diff --git a/crates/streams/src/plaineventsjson.rs b/crates/streams/src/plaineventsjson.rs index ac919bb..d599711 100644 --- a/crates/streams/src/plaineventsjson.rs +++ b/crates/streams/src/plaineventsjson.rs @@ -24,7 +24,7 @@ pub async fn plain_events_json( _cluster: &Cluster, open_bytes: OpenBoxedBytesStreamsBox, ) -> Result { - info!("plain_events_json evquery {:?}", evq); + debug!("plain_events_json evquery {:?}", evq); let deadline = Instant::now() + evq.timeout(); let stream = dyn_events_stream(evq, ch_conf, ctx, open_bytes).await?; @@ -40,7 +40,7 @@ pub async fn plain_events_json( //let stream = EventsToTimeBinnable::new(stream); //let stream = TimeBinnableToCollectable::new(stream); let stream = Box::pin(stream); - info!("plain_events_json boxed stream created"); + debug!("plain_events_json boxed stream created"); let collected = Collect::new( stream, deadline, @@ -50,9 +50,9 @@ pub async fn plain_events_json( None, ) .await?; - info!("plain_events_json collected"); + debug!("plain_events_json collected"); let jsval = serde_json::to_value(&collected)?; - info!("plain_events_json json serialized"); + debug!("plain_events_json json serialized"); Ok(jsval) } diff --git a/crates/streams/src/plaineventsstream.rs b/crates/streams/src/plaineventsstream.rs index d838d0d..d6ee21a 100644 --- a/crates/streams/src/plaineventsstream.rs +++ b/crates/streams/src/plaineventsstream.rs @@ -33,6 +33,7 @@ pub async fn dyn_events_stream( evq.transform().clone(), evq.test_do_wasm(), evq, + evq.log_level().into(), ctx, ); let inmem_bufcap = subq.inmem_bufcap(); diff --git a/crates/streams/src/tcprawclient.rs b/crates/streams/src/tcprawclient.rs index 7b5d290..f4438e4 100644 --- a/crates/streams/src/tcprawclient.rs +++ b/crates/streams/src/tcprawclient.rs @@ -203,6 +203,7 @@ pub fn make_sub_query( transform: TransformQuery, test_do_wasm: Option<&str>, sub: SUB, + log_level: String, ctx: &ReqCtx, ) -> EventsSubQuery where @@ -213,6 +214,6 @@ where select.set_wasm1(wasm1.into()); } let settings = sub.into(); - let subq = EventsSubQuery::from_parts(select, settings, ctx.reqid().into()); + let subq = EventsSubQuery::from_parts(select, settings, ctx.reqid().into(), log_level); subq } diff --git a/crates/streams/src/timebinnedjson.rs b/crates/streams/src/timebinnedjson.rs index 77410cb..caf7677 100644 --- a/crates/streams/src/timebinnedjson.rs +++ b/crates/streams/src/timebinnedjson.rs @@ -50,6 +50,7 @@ async fn timebinnable_stream( query.transform().clone(), query.test_do_wasm(), &query, + query.log_level().into(), ctx, ); let inmem_bufcap = subq.inmem_bufcap(); diff --git a/crates/taskrun/src/taskrun.rs b/crates/taskrun/src/taskrun.rs index 29d36bd..02f1509 100644 --- a/crates/taskrun/src/taskrun.rs +++ b/crates/taskrun/src/taskrun.rs @@ -8,6 +8,7 @@ use err::Error; use std::fmt; use std::future::Future; use std::io; +use std::marker::PhantomData; use std::panic; use std::sync::Arc; use std::sync::Mutex; @@ -40,7 +41,7 @@ fn on_thread_start() { format!("unknown payload type") }; error!( - "✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗✗ panicking\n{:?}\nLOCATION: {:?}\nPAYLOAD: {:?}\ninfo object: {:?}\nerr: {:?}", + "panicking\n{:?}\nLOCATION: {:?}\nPAYLOAD: {:?}\ninfo object: {:?}\nerr: {:?}", Error::with_msg("catched panic in taskrun::run"), info.location(), info.payload(), @@ -105,6 +106,37 @@ where } } +struct LogFilterLayer +where + L: tracing_subscriber::Layer, + S: tracing::Subscriber, +{ + name: String, + inner: L, + _ph1: PhantomData, +} + +impl LogFilterLayer +where + L: tracing_subscriber::Layer, + S: tracing::Subscriber, +{ + fn new(name: String, inner: L) -> Self { + Self { + name, + inner, + _ph1: PhantomData, + } + } +} + +impl tracing_subscriber::Layer for LogFilterLayer +where + L: tracing_subscriber::Layer, + S: tracing::Subscriber, +{ +} + fn tracing_init_inner(mode: TracingMode) -> Result<(), Error> { use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; @@ -131,6 +163,19 @@ fn tracing_init_inner(mode: TracingMode) -> Result<(), Error> { .with_default_directive(tracing::metadata::LevelFilter::INFO.into()) .from_env() .map_err(|e| Error::with_msg_no_trace(format!("can not build tracing env filter {e}")))?; + let filter_2 = tracing_subscriber::EnvFilter::builder() + .with_env_var("RUST_LOG_2") + .with_default_directive(tracing::metadata::LevelFilter::INFO.into()) + .from_env() + .map_err(|e| Error::with_msg_no_trace(format!("can not build tracing env filter {e}")))?; + // let filter_3 = tracing_subscriber::filter::dynamic_filter_fn(|meta, ctx| { + // // + // if ["scyllaconn"].contains(&meta.target()) { + // true + // } else { + // true + // } + // }); let fmt_layer = tracing_subscriber::fmt::Layer::new() .with_writer(io::stderr) .with_timer(timer) @@ -138,7 +183,12 @@ fn tracing_init_inner(mode: TracingMode) -> Result<(), Error> { .with_ansi(false) .with_thread_names(true) .event_format(formatter::FormatTxt) - .with_filter(filter); + .with_filter(filter_2) + .with_filter(filter) + // .and_then(LogFilterLayer::new("lay1".into())) + // .and_then(LogFilterLayer::new("lay2".into())) + ; + // let layer_2 = LogFilterLayer::new("lay1".into(), fmt_layer); let reg = tracing_subscriber::registry();