daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Write bins, support config reload

Browse Source
This commit is contained in:
Dominik Werder
2024-10-10 12:27:34 +02:00
parent d4b8beaa82
commit 87e6dfdcaa
16 changed files with 938 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
scywr/src/insertworker.rs
View File

@@ -8,6 +8,7 @@ use crate::iteminsertqueue::InsertItem;
use crate::iteminsertqueue::MspItem;
use crate::iteminsertqueue::QueryItem;
use crate::iteminsertqueue::TimeBinSimpleF32;
use crate::iteminsertqueue::TimeBinSimpleF32V01;
use crate::store::DataStore;
use async_channel::Receiver;
use atomic::AtomicU64;
@@ -274,6 +275,9 @@ where
QueryItem::Insert(item) => prepare_query_insert_futs(item, &data_store, &stats, tsnow),
QueryItem::Msp(item) => prepare_msp_insert_futs(item, &data_store, &stats, tsnow),
QueryItem::TimeBinSimpleF32(item) => prepare_timebin_insert_futs(item, &data_store, &stats, tsnow),
QueryItem::TimeBinSimpleF32V01(item) => {
prepare_timebin_v01_insert_futs(item, &data_store, &stats, tsnow)
}
QueryItem::Accounting(item) => prepare_accounting_insert_futs(item, &data_store, &stats, tsnow),
QueryItem::AccountingRecv(item) => {
prepare_accounting_recv_insert_futs(item, &data_store, &stats, tsnow)
@@ -305,6 +309,9 @@ fn inspect_items(
QueryItem::TimeBinSimpleF32(_) => {
trace_item_execute!("execute {worker_name} TimeBinSimpleF32");
}
QueryItem::TimeBinSimpleF32V01(_) => {
trace_item_execute!("execute {worker_name} TimeBinSimpleF32V01");
}
QueryItem::Accounting(_) => {
trace_item_execute!("execute {worker_name} Accounting {item:?}");
}
@@ -400,6 +407,48 @@ fn prepare_timebin_insert_futs(
futs
}
fn prepare_timebin_v01_insert_futs(
item: TimeBinSimpleF32V01,
data_store: &Arc<DataStore>,
stats: &Arc<InsertWorkerStats>,
tsnow: Instant,
) -> SmallVec<[InsertFut; 4]> {
trace!("have time bin patch to insert: {item:?}");
let params = (
item.series.id() as i64,
item.bin_len_ms,
item.ts_msp.to_i64(),
item.off,
item.count,
item.min,
item.max,
item.avg,
);
// TODO would be better to count inserts only on completed insert
stats.inserted_binned().inc();
let fut = InsertFut::new(
data_store.scy.clone(),
data_store.qu_insert_binned_scalar_f32_v01.clone(),
params,
tsnow,
stats.clone(),
);
let futs = smallvec![fut];
// TODO match on the query result:
// match qres {
// Ok(_) => {
// backoff = backoff_0;
// }
// Err(e) => {
// stats_inc_for_err(&stats, &crate::iteminsertqueue::Error::QueryError(e));
// back_off_sleep(&mut backoff).await;
// }
// }
futs
}
fn prepare_accounting_insert_futs(
item: Accounting,
data_store: &Arc<DataStore>,

56
scywr/src/iteminsertqueue.rs
View File

@@ -98,6 +98,25 @@ impl ScalarValue {
ScalarValue::Bool(x) => x.to_string(),
}
}
pub fn f32_for_binning(&self) -> f32 {
use ScalarValue::*;
match self {
U8(x) => *x as f32,
U16(x) => *x as f32,
U32(x) => *x as f32,
U64(x) => *x as f32,
I8(x) => *x as f32,
I16(x) => *x as f32,
I32(x) => *x as f32,
I64(x) => *x as f32,
F32(x) => *x as f32,
F64(x) => *x as f32,
Enum(x, _) => *x as f32,
String(x) => x.len() as f32,
Bool(x) => f32::from(*x),
}
}
}
#[derive(Clone, Debug, PartialEq)]
@@ -294,6 +313,23 @@ impl ArrayValue {
Bool(x) => format!("{}", x.get(0).map_or(false, |x| *x)),
}
}
pub fn f32_for_binning(&self) -> f32 {
use ArrayValue::*;
match self {
U8(x) => x.iter().fold(0., |a, x| a + *x as f32),
U16(x) => x.iter().fold(0., |a, x| a + *x as f32),
U32(x) => x.iter().fold(0., |a, x| a + *x as f32),
U64(x) => x.iter().fold(0., |a, x| a + *x as f32),
I8(x) => x.iter().fold(0., |a, x| a + *x as f32),
I16(x) => x.iter().fold(0., |a, x| a + *x as f32),
I32(x) => x.iter().fold(0., |a, x| a + *x as f32),
I64(x) => x.iter().fold(0., |a, x| a + *x as f32),
F32(x) => x.iter().fold(0., |a, x| a + *x as f32),
F64(x) => x.iter().fold(0., |a, x| a + *x as f32),
Bool(x) => x.iter().fold(0., |a, x| a + f32::from(*x)),
}
}
}
#[derive(Clone, Debug, PartialEq)]
@@ -323,6 +359,13 @@ impl DataValue {
DataValue::Array(x) => x.string_short(),
}
}
pub fn f32_for_binning(&self) -> f32 {
match self {
DataValue::Scalar(x) => x.f32_for_binning(),
DataValue::Array(x) => x.f32_for_binning(),
}
}
}
pub trait GetValHelp<T> {
@@ -506,12 +549,25 @@ pub struct TimeBinSimpleF32 {
pub avg: f32,
}
#[derive(Debug, Clone)]
pub struct TimeBinSimpleF32V01 {
pub series: SeriesId,
pub bin_len_ms: i32,
pub ts_msp: TsMs,
pub off: i32,
pub count: i64,
pub min: f32,
pub max: f32,
pub avg: f32,
}
// Needs to be Clone to send it to multiple retention times if required.
#[derive(Debug, Clone)]
pub enum QueryItem {
Insert(InsertItem),
Msp(MspItem),
TimeBinSimpleF32(TimeBinSimpleF32),
TimeBinSimpleF32V01(TimeBinSimpleF32V01),
Accounting(Accounting),
AccountingRecv(AccountingRecv),
}

22
scywr/src/schema.rs
View File

@@ -674,6 +674,28 @@ pub async fn migrate_scylla_data_schema(
);
tab.setup(do_change, scy).await?;
}
{
let tab = GenTwcsTab::new(
ks,
rett.table_prefix(),
"binned_scalar_f32_v01",
&[
("series", "bigint"),
("bin_len_ms", "int"),
("ts_msp", "bigint"),
("off", "int"),
("count", "bigint"),
("min", "float"),
("max", "float"),
("avg", "float"),
],
["series", "bin_len_ms", "ts_msp"],
["off"],
rett.ttl_binned(),
);
let do_change = true;
tab.setup(do_change, scy).await?;
}
{
let tab = GenTwcsTab::new(
ks,

10
scywr/src/store.rs
View File

@@ -45,6 +45,7 @@ pub struct DataStore {
pub qu_insert_array_f32: Arc<PreparedStatement>,
pub qu_insert_array_f64: Arc<PreparedStatement>,
pub qu_insert_array_bool: Arc<PreparedStatement>,
pub qu_insert_binned_scalar_f32_v01: Arc<PreparedStatement>,
pub qu_insert_binned_scalar_f32_v02: Arc<PreparedStatement>,
pub qu_account_00: Arc<PreparedStatement>,
pub qu_account_recv_00: Arc<PreparedStatement>,
@@ -149,6 +150,14 @@ impl DataStore {
let qu_insert_array_f64 = prep_qu_ins_b!("events_array_f64", rett, scy);
let qu_insert_array_bool = prep_qu_ins_b!("events_array_bool", rett, scy);
let qu_insert_binned_scalar_f32_v01 = prep_qu_ins_c!(
"binned_scalar_f32_v01",
"series, bin_len_ms, ts_msp, off, count, min, max, avg",
"?, ?, ?, ?, ?, ?, ?, ?",
rett,
scy
);
let qu_insert_binned_scalar_f32_v02 = prep_qu_ins_c!(
"binned_scalar_f32",
"series, bin_len_ms, ts_msp, off, count, min, max, avg",
@@ -210,6 +219,7 @@ impl DataStore {
qu_insert_array_f32,
qu_insert_array_f64,
qu_insert_array_bool,
qu_insert_binned_scalar_f32_v01,
qu_insert_binned_scalar_f32_v02,
qu_account_00,
qu_account_recv_00,