daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Udate deps, backend list for plain node

Browse Source
This commit is contained in:
Dominik Werder
2024-06-12 13:43:28 +02:00
parent 437e6d0d76
commit 03e8ac7a70
35 changed files with 1459 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scyllaconn/src/accounting/toplist.rs
+4 -4
View File
@@ -43,14 +43,14 @@ pub async fn read_ts(ts: u64, scy: Arc<ScySession>) -> Result<UsageData, Error>
let snap = EMIT_ACCOUNTING_SNAP.ms() / 1000;
info!("ts {ts} snap {snap:?}");
let ts = ts / timeunits::SEC / snap * snap;
let cql = concat!("select series, count, bytes from lt_account_00 where part = ? and ts = ?");
let qu = prep(cql, scy.clone()).await?;
let ret = read_ts_inner(ts, qu, scy).await?;
let ret = read_ts_inner(ts, scy).await?;
Ok(ret)
}
async fn read_ts_inner(ts: u64, qu: PreparedStatement, scy: Arc<ScySession>) -> Result<UsageData, Error> {
async fn read_ts_inner(ts: u64, scy: Arc<ScySession>) -> Result<UsageData, Error> {
type RowType = (i64, i64, i64);
let cql = concat!("select series, count, bytes from lt_account_00 where part = ? and ts = ?");
let qu = prep(cql, scy.clone()).await?;
let mut ret = UsageData::new(ts);
for part in 0..255_u32 {
let mut res = scy
crates/scyllaconn/src/events.rs
+72 -14
View File
@@ -224,6 +224,36 @@ macro_rules! impl_scaty_array {
};
}
impl ValTy for Vec<String> {
type ScaTy = String;
type ScyTy = Vec<String>;
type Container = EventsDim1<String>;
fn from_scyty(inp: Self::ScyTy) -> Self {
inp
}
fn from_valueblob(inp: Vec<u8>) -> Self {
todo!()
}
fn table_name() -> &'static str {
"st_events_array_enum"
}
fn default() -> Self {
Vec::new()
}
fn is_valueblob() -> bool {
false
}
fn st_name() -> &'static str {
"enum"
}
}
impl_scaty_scalar!(u8, i8, "u8", "st_events_scalar_u8");
impl_scaty_scalar!(u16, i16, "u16", "st_events_scalar_u16");
impl_scaty_scalar!(u32, i32, "u32", "st_events_scalar_u32");
@@ -248,7 +278,6 @@ impl_scaty_array!(Vec<i64>, i64, Vec<i64>, "i64", "st_events_array_i64");
impl_scaty_array!(Vec<f32>, f32, Vec<f32>, "f32", "st_events_array_f32");
impl_scaty_array!(Vec<f64>, f64, Vec<f64>, "f64", "st_events_array_f64");
impl_scaty_array!(Vec<bool>, bool, Vec<bool>, "bool", "st_events_array_bool");
// impl_scaty_array!(Vec<String>, String, Vec<String>, "string", "st_events_array_string");
struct ReadNextValuesOpts {
series: u64,
@@ -307,14 +336,19 @@ where
ts_lsp_max,
table_name,
);
let dir = "fwd";
let qu_name = if opts.with_values {
if ST::is_valueblob() {
format!("array_{}_valueblobs_fwd", ST::st_name())
format!("array_{}_valueblobs_{}", ST::st_name(), dir)
} else {
format!("array_{}_values_fwd", ST::st_name())
format!("scalar_{}_values_{}", ST::st_name(), dir)
}
} else {
format!("array_{}_timestamps_fwd", ST::st_name())
if ST::is_valueblob() {
format!("array_{}_timestamps_{}", ST::st_name(), dir)
} else {
format!("scalar_{}_timestamps_{}", ST::st_name(), dir)
}
};
let qu = stmts.read_value_queries.get(&qu_name).ok_or_else(|| {
let e = Error::with_msg_no_trace(format!("can not find query name {}", qu_name));
@@ -343,14 +377,19 @@ where
DtNano::from_ns(0)
};
trace!("BCK ts_msp {} ts_lsp_max {} {}", ts_msp, ts_lsp_max, table_name,);
let dir = "bck";
let qu_name = if opts.with_values {
if ST::is_valueblob() {
format!("array_{}_valueblobs_bck", ST::st_name())
format!("array_{}_valueblobs_{}", ST::st_name(), dir)
} else {
format!("array_{}_values_bck", ST::st_name())
format!("scalar_{}_values_{}", ST::st_name(), dir)
}
} else {
format!("array_{}_timestamps_bck", ST::st_name())
if ST::is_valueblob() {
format!("array_{}_timestamps_{}", ST::st_name(), dir)
} else {
format!("scalar_{}_timestamps_{}", ST::st_name(), dir)
}
};
let qu = stmts.read_value_queries.get(&qu_name).ok_or_else(|| {
let e = Error::with_msg_no_trace(format!("can not find query name {}", qu_name));
@@ -512,6 +551,7 @@ impl ReadValues {
ScalarType::F64 => read_next_values::<f64>(opts).await,
ScalarType::BOOL => read_next_values::<bool>(opts).await,
ScalarType::STRING => read_next_values::<String>(opts).await,
ScalarType::Enum => read_next_values::<String>(opts).await,
ScalarType::ChannelStatus => {
warn!("read scalar channel status not yet supported");
err::todoval()
@@ -533,6 +573,7 @@ impl ReadValues {
warn!("read array string not yet supported");
err::todoval()
}
ScalarType::Enum => read_next_values::<Vec<String>>(opts).await,
ScalarType::ChannelStatus => {
warn!("read array channel status not yet supported");
err::todoval()
@@ -572,6 +613,7 @@ pub struct EventsStreamScylla {
found_one_after: bool,
with_values: bool,
outqueue: VecDeque<Box<dyn Events>>,
ts_seen_max: u64,
}
impl EventsStreamScylla {
@@ -600,6 +642,7 @@ impl EventsStreamScylla {
found_one_after: false,
with_values,
outqueue: VecDeque::new(),
ts_seen_max: 0,
}
}
@@ -618,7 +661,7 @@ impl EventsStreamScylla {
trace!("ts_msp_bck {:?}", self.ts_msp_bck);
trace!("ts_msp_fwd {:?}", self.ts_msp_fwd);
if let Some(msp) = self.ts_msp_bck.pop_back() {
trace!("Try ReadBack1");
trace!("start ReadBack1 msp {}", msp);
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -631,7 +674,7 @@ impl EventsStreamScylla {
);
self.state = FrState::ReadBack1(st);
} else if self.ts_msp_fwd.len() > 0 {
trace!("Go straight for forward read");
trace!("begin immediately with forward read");
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -653,6 +696,7 @@ impl EventsStreamScylla {
if item.len() > 0 {
self.outqueue.push_back(item);
if self.ts_msp_fwd.len() > 0 {
trace!("start forward read after back1");
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -669,7 +713,7 @@ impl EventsStreamScylla {
}
} else {
if let Some(msp) = self.ts_msp_bck.pop_back() {
trace!("Try ReadBack2");
trace!("start ReadBack2 msp {}", msp);
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -682,7 +726,7 @@ impl EventsStreamScylla {
);
self.state = FrState::ReadBack2(st);
} else if self.ts_msp_fwd.len() > 0 {
trace!("No 2nd back MSP, go for forward read");
trace!("no 2nd back MSP, go for forward read");
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -695,18 +739,19 @@ impl EventsStreamScylla {
);
self.state = FrState::ReadValues(st);
} else {
trace!("No 2nd back MSP, but also nothing to go forward");
trace!("no 2nd back msp, but also nothing to go forward");
self.state = FrState::DataDone;
}
}
}
fn back_2_done(&mut self, item: Box<dyn Events>) {
trace!("back_1_done item len {}", item.len());
trace!("back_2_done item len {}", item.len());
if item.len() > 0 {
self.outqueue.push_back(item);
}
if self.ts_msp_fwd.len() > 0 {
trace!("start forward read after back2");
let st = ReadValues::new(
self.series,
self.scalar_type.clone(),
@@ -719,6 +764,7 @@ impl EventsStreamScylla {
);
self.state = FrState::ReadValues(st);
} else {
trace!("nothing to forward read after back 2");
self.state = FrState::DataDone;
}
}
@@ -745,7 +791,19 @@ impl Stream for EventsStreamScylla {
loop {
if let Some(item) = self.outqueue.pop_front() {
item.verify();
item.output_info();
if let Some(item_min) = item.ts_min() {
if item_min < self.ts_seen_max {
debug!("ordering error A {} {}", item_min, self.ts_seen_max);
}
}
if let Some(item_max) = item.ts_max() {
if item_max < self.ts_seen_max {
debug!("ordering error B {} {}", item_max, self.ts_seen_max);
} else {
self.ts_seen_max = item_max;
}
}
debug!("deliver item {}", item.output_info());
break Ready(Some(Ok(ChannelEvents::Events(item))));
}
break match self.state {
crates/scyllaconn/src/worker.rs
+19 -13
View File
@@ -100,28 +100,34 @@ impl ScyllaQueue {
#[derive(Debug)]
pub struct ScyllaWorker {
rx: Receiver<Job>,
scy: Arc<Session>,
stmts_st: Arc<StmtsEventsRt>,
scyconf_st: ScyllaConfig,
scyconf_mt: ScyllaConfig,
scyconf_lt: ScyllaConfig,
}
impl ScyllaWorker {
pub async fn new(
scyconf_st: &ScyllaConfig,
scyconf_mt: &ScyllaConfig,
scyconf_lt: &ScyllaConfig,
scyconf_st: ScyllaConfig,
scyconf_mt: ScyllaConfig,
scyconf_lt: ScyllaConfig,
) -> Result<(ScyllaQueue, Self), Error> {
let (tx, rx) = async_channel::bounded(64);
let scy = create_scy_session_no_ks(scyconf_st).await?;
let scy = Arc::new(scy);
let rtpre = format!("{}.st_", scyconf_st.keyspace);
let stmts_st = StmtsEventsRt::new(&rtpre, &scy).await?;
let stmts_st = Arc::new(stmts_st);
let queue = ScyllaQueue { tx };
let worker = Self { rx, scy, stmts_st };
let worker = Self {
rx,
scyconf_st,
scyconf_mt,
scyconf_lt,
};
Ok((queue, worker))
}
pub async fn work(self) -> Result<(), Error> {
let scy = create_scy_session_no_ks(&self.scyconf_st).await?;
let scy = Arc::new(scy);
let rtpre = format!("{}.st_", self.scyconf_st.keyspace);
let stmts_st = StmtsEventsRt::new(&rtpre, &scy).await?;
let stmts_st = Arc::new(stmts_st);
loop {
let x = self.rx.recv().await;
let job = match x {
@@ -133,13 +139,13 @@ impl ScyllaWorker {
};
match job {
Job::FindTsMsp(series, range, tx) => {
let res = crate::events::find_ts_msp_worker(series, range, &self.stmts_st, &self.scy).await;
let res = crate::events::find_ts_msp_worker(series, range, &stmts_st, &scy).await;
if tx.send(res.map_err(Into::into)).await.is_err() {
// TODO count for stats
}
}
Job::ReadNextValues(job) => {
let fut = (job.futgen)(self.scy.clone(), self.stmts_st.clone());
let fut = (job.futgen)(scy.clone(), stmts_st.clone());
let res = fut.await;
if job.tx.send(res.map_err(Into::into)).await.is_err() {
// TODO count for stats