daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update scylla dependency

Browse Source
This commit is contained in:
Dominik Werder
2024-12-05 17:45:15 +01:00
parent 8aa922a6a1
commit e85728c1e3
14 changed files with 189 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scyllaconn/src/accounting/toplist.rs
+4 -3
View File
@@ -2,6 +2,7 @@ use daqbuf_err as err;
use err::thiserror;
use err::ThisError;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use netpod::log::*;
use netpod::ttl::RetentionTime;
use netpod::TsMs;
@@ -14,6 +15,7 @@ use scylla::Session as ScySession;
pub enum Error {
ScyllaQuery(#[from] scylla::transport::errors::QueryError),
ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError),
ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError),
UsageDataMalformed,
}
@@ -118,9 +120,8 @@ async fn read_ts_inner(ks: &str, rt: RetentionTime, ts: TsMs, scy: &ScySession)
let mut res = scy
.execute_iter(qu.clone(), (part as i32, ts_sec))
.await?
.into_typed::<RowType>();
while let Some(row) = res.next().await {
let row = row?;
.rows_stream::<RowType>()?;
while let Some(row) = res.try_next().await? {
let series = row.0 as u64;
let count = row.1 as u64;
let bytes = row.2 as u64;
crates/scyllaconn/src/accounting/totals.rs
+4 -4
View File
@@ -4,7 +4,7 @@ use err::Error;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use items_0::Empty;
use items_0::Extendable;
use items_0::WithLen;
@@ -36,12 +36,12 @@ async fn read_next(
scy.execute_iter(qu.clone(), (part as i32, ts_msp as i64))
.await
.err_conv()?
.into_typed::<RowType>()
.rows_stream::<RowType>()
.err_conv()?
} else {
return Err(Error::with_msg_no_trace("no backward support"));
};
while let Some(row) = res.next().await {
let row = row.map_err(Error::from_string)?;
while let Some(row) = res.try_next().await.err_conv()? {
let _ts = ts_msp;
let _series = row.0 as u64;
let count = row.1 as u64;
crates/scyllaconn/src/bincache.rs
+10 -4
View File
@@ -1,6 +1,7 @@
use crate::events2::prepare::StmtsCache;
use crate::worker::ScyllaQueue;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use items_0::timebin::BinsBoxed;
use items_2::binning::container_bins::ContainerBins;
use netpod::DtMs;
@@ -63,7 +64,7 @@ pub async fn worker_write(
lst,
);
// trace!("cache write {:?}", params);
scy.execute(stmts_cache.st_write_f32(), params)
scy.execute_unpaged(stmts_cache.st_write_f32(), params)
.await
.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
}
@@ -90,10 +91,15 @@ pub async fn worker_read(
.execute_iter(stmts_cache.st_read_f32().clone(), params)
.await
.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
let mut it = res.into_typed::<(i32, i64, f32, f32, f32, f32)>();
let mut it = res
.rows_stream::<(i32, i64, f32, f32, f32, f32)>()
.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
let mut bins = ContainerBins::new();
while let Some(x) = it.next().await {
let row = x.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?;
while let Some(row) = it
.try_next()
.await
.map_err(|e| streams::timebin::cached::reader::Error::Scylla(e.to_string()))?
{
let off = row.0 as u64;
let cnt = row.1 as u64;
let min = row.2;
crates/scyllaconn/src/errconv.rs
+1 -2
View File
@@ -3,7 +3,6 @@ use err::Error;
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
use scylla::transport::errors::NewSessionError as ScyNewSessionError;
use scylla::transport::errors::QueryError as ScyQueryError;
use scylla::transport::query_result::RowsExpectedError;
pub trait ErrConv<T> {
fn err_conv(self) -> Result<T, Error>;
@@ -44,7 +43,7 @@ impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
}
}
impl<T> ErrConv<T> for Result<T, RowsExpectedError> {
impl<T> ErrConv<T> for Result<T, scylla::deserialize::TypeCheckError> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
crates/scyllaconn/src/events.rs
+117 -71
View File
@@ -9,6 +9,7 @@ use err::thiserror;
use err::ThisError;
use futures_util::Future;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use items_0::scalar_ops::ScalarOps;
use items_0::timebin::BinningggContainerEventsDyn;
use items_0::Appendable;
@@ -38,6 +39,7 @@ pub enum Error {
ScyllaNextRow(#[from] scylla::transport::iterator::NextRowError),
ScyllaTypeConv(#[from] scylla::cql_to_rust::FromRowError),
ScyllaWorker(Box<crate::worker::Error>),
ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError),
MissingQuery(String),
NotTokenAware,
RangeEndOverflow,
@@ -55,7 +57,7 @@ impl From<crate::worker::Error> for Error {
pub(super) trait ValTy: Sized + 'static {
type ScaTy: ScalarOps + std::default::Default;
type ScyTy: scylla::cql_to_rust::FromCqlVal<scylla::frame::response::result::CqlValue>;
type ScyTy: for<'a, 'b> scylla::deserialize::DeserializeValue<'a, 'b>;
type Container: BinningggContainerEventsDyn + Empty + Appendable<Self>;
fn from_scyty(inp: Self::ScyTy) -> Self;
fn from_valueblob(inp: Vec<u8>) -> Self;
@@ -521,21 +523,21 @@ where
jobtrace.add_event_now(ReadEventKind::CallExecuteIter);
let mut res = scy.execute_iter(qu.clone(), params).await?;
if use_method_2 == false {
let mut rows = Vec::new();
while let Some(x) = res.next().await {
rows.push(x?);
}
let mut last_before = None;
let ret = <ST as ValTy>::convert_rows(rows, range, ts_msp, with_values, !opts.fwd, &mut last_before)?;
ret
// let mut rows = Vec::new();
// while let Some(x) = res.next().await {
// rows.push(x?);
// }
// let mut last_before = None;
// let ret = <ST as ValTy>::convert_rows(rows, range, ts_msp, with_values, !opts.fwd, &mut last_before)?;
// ret
todo!()
} else {
let mut ret = <ST as ValTy>::Container::empty();
// TODO must branch already here depending on what input columns we expect
if with_values {
if <ST as ValTy>::is_valueblob() {
let mut it = res.into_typed::<(i64, Vec<u8>)>();
while let Some(x) = it.next().await {
let row = x?;
let mut it = res.rows_stream::<(i64, Vec<u8>)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::from_valueblob(row.1);
ret.push(ts, value);
@@ -543,9 +545,8 @@ where
ret
} else {
let mut i = 0;
let mut it = res.into_typed::<(i64, ST::ScyTy)>();
while let Some(x) = it.next().await {
let row = x?;
let mut it = res.rows_stream::<(i64, ST::ScyTy)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::from_scyty(row.1);
ret.push(ts, value);
@@ -560,9 +561,8 @@ where
ret
}
} else {
let mut it = res.into_typed::<(i64,)>();
while let Some(x) = it.next().await {
let row = x?;
let mut it = res.rows_stream::<(i64,)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::default();
ret.push(ts, value);
@@ -590,16 +590,51 @@ where
let params = (series as i64, ts_msp.ms() as i64, ts_lsp_max.ns() as i64);
trace!("BCK event search params {:?}", params);
let mut res = scy.execute_iter(qu.clone(), params).await?;
let mut rows = Vec::new();
while let Some(x) = res.next().await {
rows.push(x?);
{
let mut ret = <ST as ValTy>::Container::empty();
// TODO must branch already here depending on what input columns we expect
if with_values {
if <ST as ValTy>::is_valueblob() {
let mut it = res.rows_stream::<(i64, Vec<u8>)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::from_valueblob(row.1);
ret.push(ts, value);
}
ret
} else {
let mut i = 0;
let mut it = res.rows_stream::<(i64, ST::ScyTy)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::from_scyty(row.1);
ret.push(ts, value);
i += 1;
if i % 2000 == 0 {
jobtrace.add_event_now(ReadEventKind::ScyllaReadRow(i));
}
}
{
jobtrace.add_event_now(ReadEventKind::ScyllaReadRowDone(i));
}
ret
}
} else {
let mut it = res.rows_stream::<(i64,)>()?;
while let Some(row) = it.try_next().await? {
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = <ST as ValTy>::default();
ret.push(ts, value);
}
ret
}
}
let mut _last_before = None;
let ret = ST::convert_rows(rows, range, ts_msp, with_values, !opts.fwd, &mut _last_before)?;
if ret.len() > 1 {
error!("multiple events in backwards search {}", ret.len());
}
ret
// let mut _last_before = None;
// let ret = ST::convert_rows(rows, range, ts_msp, with_values, !opts.fwd, &mut _last_before)?;
// if ret.len() > 1 {
// error!("multiple events in backwards search {}", ret.len());
// }
// ret
};
trace!("read ts_msp {} len {}", ts_msp.fmt(), ret.len());
let ret = Box::new(ret);
@@ -614,53 +649,64 @@ fn convert_rows_0<ST: ValTy>(
bck: bool,
last_before: &mut Option<(TsNano, ST)>,
) -> Result<<ST as ValTy>::Container, Error> {
let mut ret = <ST as ValTy>::Container::empty();
for row in rows {
let (ts, value) = if with_values {
if ST::is_valueblob() {
let row: (i64, Vec<u8>) = row.into_typed()?;
// trace!("read a value blob len {}", row.1.len());
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = ValTy::from_valueblob(row.1);
(ts, value)
} else {
let row: (i64, ST::ScyTy) = row.into_typed()?;
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = ValTy::from_scyty(row.1);
(ts, value)
}
} else {
let row: (i64,) = row.into_typed()?;
let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
let value = ValTy::default();
(ts, value)
};
if bck {
if ts >= range.beg() {
// TODO count as logic error
error!("ts >= range.beg");
} else if ts < range.beg() {
ret.push(ts, value);
} else {
*last_before = Some((ts, value));
}
} else {
if ts >= range.end() {
// TODO count as logic error
error!("ts >= range.end");
} else if ts >= range.beg() {
ret.push(ts, value);
} else {
if last_before.is_none() {
warn!("encounter event before range in forward read {ts}");
}
*last_before = Some((ts, value));
}
}
}
Ok(ret)
todo!()
}
// fn convert_rows_0<ST: ValTy>(
// rows: Vec<Row>,
// range: ScyllaSeriesRange,
// ts_msp: TsMs,
// with_values: bool,
// bck: bool,
// last_before: &mut Option<(TsNano, ST)>,
// ) -> Result<<ST as ValTy>::Container, Error> {
// let mut ret = <ST as ValTy>::Container::empty();
// for row in rows {
// let (ts, value) = if with_values {
// if ST::is_valueblob() {
// let row: (i64, Vec<u8>) = row.into_typed()?;
// // trace!("read a value blob len {}", row.1.len());
// let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
// let value = ValTy::from_valueblob(row.1);
// (ts, value)
// } else {
// let row: (i64, ST::ScyTy) = row.into_typed()?;
// let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
// let value = ValTy::from_scyty(row.1);
// (ts, value)
// }
// } else {
// let row: (i64,) = row.into_typed()?;
// let ts = TsNano::from_ns(ts_msp.ns_u64() + row.0 as u64);
// let value = ValTy::default();
// (ts, value)
// };
// if bck {
// if ts >= range.beg() {
// // TODO count as logic error
// error!("ts >= range.beg");
// } else if ts < range.beg() {
// ret.push(ts, value);
// } else {
// *last_before = Some((ts, value));
// }
// } else {
// if ts >= range.end() {
// // TODO count as logic error
// error!("ts >= range.end");
// } else if ts >= range.beg() {
// ret.push(ts, value);
// } else {
// if last_before.is_none() {
// warn!("encounter event before range in forward read {ts}");
// }
// *last_before = Some((ts, value));
// }
// }
// }
// Ok(ret)
// }
fn convert_rows_enum(
rows: Vec<Row>,
range: ScyllaSeriesRange,
crates/scyllaconn/src/events2/events.rs
-5
View File
@@ -466,12 +466,7 @@ impl Stream for EventsStreamRt {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
let mut i = 0usize;
loop {
i += 1;
if i > 500000000000 {
panic!("too many iterations")
}
if let Some(mut item) = self.out.pop_front() {
if item.is_consistent() == false {
warn_item!("{}bad item {:?}", "\n\n--------------------------\n", item);
crates/scyllaconn/src/events2/msp.rs
+6 -7
View File
@@ -8,7 +8,7 @@ use err::ThisError;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use netpod::log::*;
use netpod::ttl::RetentionTime;
use netpod::TsMs;
@@ -28,6 +28,7 @@ pub enum Error {
Worker(Box<crate::worker::Error>),
ScyllaQuery(#[from] scylla::transport::errors::QueryError),
ScyllaRow(#[from] scylla::transport::iterator::NextRowError),
ScyllaTypeCheck(#[from] scylla::deserialize::TypeCheckError),
}
impl From<crate::worker::Error> for Error {
@@ -331,9 +332,8 @@ async fn find_ts_msp_fwd(
let mut res = scy
.execute_iter(stmts.rt(rt).ts_msp_fwd().clone(), params)
.await?
.into_typed::<(i64,)>();
while let Some(x) = res.next().await {
let row = x?;
.rows_stream::<(i64,)>()?;
while let Some(row) = res.try_next().await? {
let ts = TsMs::from_ms_u64(row.0 as u64);
ret.push_back(ts);
}
@@ -352,9 +352,8 @@ async fn find_ts_msp_bck(
let mut res = scy
.execute_iter(stmts.rt(rt).ts_msp_bck().clone(), params)
.await?
.into_typed::<(i64,)>();
while let Some(x) = res.next().await {
let row = x?;
.rows_stream::<(i64,)>()?;
while let Some(row) = res.try_next().await? {
let ts = TsMs::from_ms_u64(row.0 as u64);
ret.push_front(ts);
}
crates/scyllaconn/src/schema.rs
+1 -1
View File
@@ -19,6 +19,6 @@ pub async fn schema(rt: RetentionTime, scyco: &ScyllaConfig, scy: &ScySession) -
rt.table_prefix(),
table
);
let _ = scy.query(cql, ()).await;
let _ = scy.query_unpaged(cql, ()).await;
Ok(())
}
crates/scyllaconn/src/status.rs
+5 -4
View File
@@ -4,6 +4,7 @@ use err::Error;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::TryStreamExt;
use items_0::isodate::IsoDateTime;
use items_0::Empty;
use items_0::Extendable;
@@ -53,7 +54,7 @@ async fn read_next_status_events(
let cql = concat!(
"select ts_lsp, kind from channel_status where series = ? and ts_msp = ? and ts_lsp >= ? and ts_lsp < ?"
);
scy.query(
scy.query_iter(
cql,
(series as i64, ts_msp as i64, ts_lsp_min as i64, ts_lsp_max as i64),
)
@@ -73,14 +74,14 @@ async fn read_next_status_events(
let cql = concat!(
"select ts_lsp, kind from channel_status where series = ? and ts_msp = ? and ts_lsp < ? order by ts_lsp desc limit 1"
);
scy.query(cql, (series as i64, ts_msp as i64, ts_lsp_max as i64))
scy.query_iter(cql, (series as i64, ts_msp as i64, ts_lsp_max as i64))
.await
.err_conv()?
};
let mut last_before = None;
let mut ret = ChannelStatusEvents::empty();
for row in res.rows_typed_or_empty::<(i64, i32)>() {
let row = row.err_conv()?;
let mut it = res.rows_stream::<(i64, i32)>().err_conv()?;
while let Some(row) = it.try_next().await.err_conv()? {
let ts = ts_msp + row.0 as u64;
let kind = row.1 as u32;
let datetime = IsoDateTime::from_unix_millis(ts / MS);