daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Find active channels and deliver values

Browse Source
This commit is contained in:
Dominik Werder
2022-06-03 16:57:59 +02:00
parent 775650c2d8
commit 3cd1b7a640
49 changed files with 1002 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
dbconn/src/lib.rs
View File

@@ -1,6 +1,5 @@
pub mod scan;
pub mod search;
pub mod pg {
pub use tokio_postgres::{Client, Error};
}
@@ -8,15 +7,17 @@ pub mod pg {
use err::Error;
use netpod::log::*;
use netpod::{Channel, Database, NodeConfigCached};
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
use scylla::transport::errors::{NewSessionError as ScyNewSessionError, QueryError as ScyQueryError};
use std::time::Duration;
use tokio_postgres::{Client, NoTls};
trait ErrConv<T> {
fn errconv(self) -> Result<T, Error>;
fn err_conv(self) -> Result<T, Error>;
}
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
fn errconv(self) -> Result<T, Error> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg(e.to_string())),
@@ -25,13 +26,39 @@ impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
}
impl<T, A> ErrConv<T> for Result<T, async_channel::SendError<A>> {
fn errconv(self) -> Result<T, Error> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg(e.to_string())),
}
}
}
impl<T> ErrConv<T> for Result<T, ScyQueryError> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}
impl<T> ErrConv<T> for Result<T, ScyNewSessionError> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}
impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}
pub async fn delay_us(mu: u64) {
tokio::time::sleep(Duration::from_micros(mu)).await;
@@ -68,7 +95,7 @@ pub async fn channel_exists(channel: &Channel, node_config: &NodeConfigCached) -
let rows = cl
.query("select rowid from channels where name = $1::text", &[&channel.name])
.await
.errconv()?;
.err_conv()?;
debug!("channel_exists {} rows", rows.len());
for row in rows {
debug!(
@@ -89,7 +116,7 @@ pub async fn database_size(node_config: &NodeConfigCached) -> Result<u64, Error>
&[&node_config.node_config.cluster.database.name],
)
.await
.errconv()?;
.err_conv()?;
if rows.len() == 0 {
Err(Error::with_msg("could not get database size"))?;
}
@@ -115,7 +142,7 @@ pub async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, E
);
let sql = sql.as_str();
let cl = create_connection(&node_config.node_config.cluster.database).await?;
let rows = cl.query(sql, &[]).await.errconv()?;
let rows = cl.query(sql, &[]).await.err_conv()?;
let mut sizes = TableSizes { sizes: vec![] };
sizes.sizes.push((format!("table"), format!("size")));
for row in rows {
@@ -127,7 +154,7 @@ pub async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, E
pub async fn random_channel(node_config: &NodeConfigCached) -> Result<String, Error> {
let sql = "select name from channels order by rowid limit 1 offset (random() * (select count(rowid) from channels))::bigint";
let cl = create_connection(&node_config.node_config.cluster.database).await?;
let rows = cl.query(sql, &[]).await.errconv()?;
let rows = cl.query(sql, &[]).await.err_conv()?;
if rows.len() == 0 {
Err(Error::with_msg("can not get random channel"))?;
}
@@ -141,11 +168,11 @@ pub async fn insert_channel(name: String, facility: i64, dbc: &Client) -> Result
&[&facility, &name],
)
.await
.errconv()?;
.err_conv()?;
if rows[0].get::<_, i64>(0) == 0 {
let sql =
concat!("insert into channels (facility, name) values ($1, $2) on conflict (facility, name) do nothing");
dbc.query(sql, &[&facility, &name]).await.errconv()?;
dbc.query(sql, &[&facility, &name]).await.err_conv()?;
}
Ok(())
}

74
dbconn/src/scan.rs
View File

@@ -49,7 +49,7 @@ pub async fn get_node_disk_ident(node_config: &NodeConfigCached, dbc: &Client) -
let rows = dbc
.query(sql, &[&node_config.node_config.cluster.backend, &node_config.node.host])
.await
.errconv()?;
.err_conv()?;
if rows.len() != 1 {
return Err(Error::with_msg(format!(
"get_node can't find unique entry for {} {}",
@@ -73,7 +73,7 @@ pub async fn get_node_disk_ident_2(
let rows = dbc
.query(sql, &[&node_config.node_config.cluster.backend, &node_config.node.host])
.await
.errconv()?;
.err_conv()?;
if rows.len() != 1 {
return Err(Error::with_msg(format!(
"get_node can't find unique entry for {} {}",
@@ -327,16 +327,16 @@ impl Stream for UpdatedDbWithChannelNamesStream {
async fn update_db_with_channel_name_list(list: Vec<String>, backend: i64, dbc: &Client) -> Result<(), Error> {
crate::delay_io_short().await;
dbc.query("begin", &[]).await.errconv()?;
dbc.query("begin", &[]).await.err_conv()?;
for ch in list {
dbc.query(
"insert into channels (facility, name) values ($1, $2) on conflict do nothing",
&[&backend, &ch],
)
.await
.errconv()?;
.err_conv()?;
}
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
Ok(())
}
@@ -351,7 +351,7 @@ pub async fn update_db_with_channel_names(
let dbc = crate::create_connection(&db_config).await?;
let node_disk_ident = get_node_disk_ident(&node_config, &dbc).await?;
let c1 = Arc::new(RwLock::new(0u32));
dbc.query("begin", &[]).await.errconv()?;
dbc.query("begin", &[]).await.err_conv()?;
let dbc = Arc::new(dbc);
let tx = Arc::new(tx);
let base_path = &node_config
@@ -373,33 +373,33 @@ pub async fn update_db_with_channel_names(
&[&fac, &ch],
)
.await
.errconv()?;
.err_conv()?;
let c2 = {
let mut g = c1.write()?;
*g += 1;
*g
};
if c2 % 200 == 0 {
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
let ret = UpdatedDbWithChannelNames {
msg: format!("current {}", ch),
count: c2,
};
tx.send(Ok(ret)).await.errconv()?;
tx.send(Ok(ret)).await.err_conv()?;
crate::delay_io_medium().await;
dbc.query("begin", &[]).await.errconv()?;
dbc.query("begin", &[]).await.err_conv()?;
}
Ok(())
}
})
.await?;
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
let c2 = *c1.read()?;
let ret = UpdatedDbWithChannelNames {
msg: format!("all done"),
count: c2,
};
tx.send(Ok(ret)).await.errconv()?;
tx.send(Ok(ret)).await.err_conv()?;
Ok::<_, Error>(())
};
let block2 = async move {
@@ -468,15 +468,15 @@ pub async fn update_db_with_all_channel_configs(
&[&node_disk_ident.facility],
)
.await
.errconv()?;
.err_conv()?;
let mut c1 = 0;
dbc.query("begin", &[]).await.errconv()?;
dbc.query("begin", &[]).await.err_conv()?;
let mut count_inserted = 0;
let mut count_updated = 0;
for row in rows {
let rowid: i64 = row.try_get(0).errconv()?;
let _facility: i64 = row.try_get(1).errconv()?;
let channel: String = row.try_get(2).errconv()?;
let rowid: i64 = row.try_get(0).err_conv()?;
let _facility: i64 = row.try_get(1).err_conv()?;
let channel: String = row.try_get(2).err_conv()?;
match update_db_with_channel_config(
node_config,
node_disk_ident,
@@ -499,26 +499,26 @@ pub async fn update_db_with_all_channel_configs(
Ok(UpdateChannelConfigResult::Done) => {
c1 += 1;
if c1 % 200 == 0 {
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
let msg = format!(
"channel no {:6} inserted {:6} updated {:6}",
c1, count_inserted, count_updated
);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
tx.send(Ok(ret)).await.errconv()?;
dbc.query("begin", &[]).await.errconv()?;
tx.send(Ok(ret)).await.err_conv()?;
dbc.query("begin", &[]).await.err_conv()?;
}
crate::delay_io_short().await;
}
}
}
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
let msg = format!(
"ALL DONE channel no {:6} inserted {:6} updated {:6}",
c1, count_inserted, count_updated
);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
tx.send(Ok(ret)).await.errconv()?;
tx.send(Ok(ret)).await.err_conv()?;
Ok::<_, Error>(())
}
.then({
@@ -528,7 +528,7 @@ pub async fn update_db_with_all_channel_configs(
Err(e) => {
let msg = format!("Seeing error: {:?}", e);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: 0 };
tx2.send(Ok(ret)).await.errconv()?;
tx2.send(Ok(ret)).await.err_conv()?;
}
}
Ok::<_, Error>(())
@@ -551,7 +551,7 @@ pub async fn update_db_with_all_channel_configs(
pub async fn update_search_cache(node_config: &NodeConfigCached) -> Result<(), Error> {
let dbc = crate::create_connection(&node_config.node_config.cluster.database).await?;
dbc.query("select update_cache()", &[]).await.errconv()?;
dbc.query("select update_cache()", &[]).await.err_conv()?;
Ok(())
}
@@ -597,7 +597,7 @@ pub async fn update_db_with_channel_config(
&[&node_disk_ident.rowid(), &channel_id],
)
.await
.errconv()?;
.err_conv()?;
if rows.len() > 1 {
return Err(Error::with_msg("more than one row"));
}
@@ -612,7 +612,7 @@ pub async fn update_db_with_channel_config(
"insert into configs_history (rowid_original, node, channel, fileSize, parsedUntil, config, tsinsert) ",
"select rowid as rowid_original, node, channel, fileSize, parsedUntil, config, now() from configs where rowid = $1"
);
dbc.query(sql, &[&rowid]).await.errconv()?;
dbc.query(sql, &[&rowid]).await.err_conv()?;
}
//ensure!(meta.len() >= parsed_until as u64, ConfigFileOnDiskShrunk{path});
(Some(rowid), true)
@@ -635,14 +635,14 @@ pub async fn update_db_with_channel_config(
],
)
.await
.errconv()?;
.err_conv()?;
*count_inserted += 1;
}
Some(_config_id_2) => {
dbc.query(
"insert into configs (node, channel, fileSize, parsedUntil, config) values ($1, $2, $3, $4, $5) on conflict (node, channel) do update set fileSize = $3, parsedUntil = $4, config = $5",
&[&node_disk_ident.rowid(), &channel_id, &(meta.len() as i64), &(buf.len() as i64), &serde_json::to_value(config)?],
).await.errconv()?;
).await.err_conv()?;
*count_updated += 1;
}
}
@@ -662,25 +662,25 @@ pub async fn update_db_with_all_channel_datafiles(
&[&node_disk_ident.facility()],
)
.await
.errconv()?;
.err_conv()?;
let mut c1 = 0;
dbc.query("begin", &[]).await.errconv()?;
dbc.query("begin", &[]).await.err_conv()?;
for row in rows {
let rowid: i64 = row.try_get(0).errconv()?;
let _facility: i64 = row.try_get(1).errconv()?;
let channel: String = row.try_get(2).errconv()?;
let rowid: i64 = row.try_get(0).err_conv()?;
let _facility: i64 = row.try_get(1).err_conv()?;
let channel: String = row.try_get(2).err_conv()?;
update_db_with_channel_datafiles(node_config, node_disk_ident, ks_prefix, rowid, &channel, dbc.clone()).await?;
c1 += 1;
if c1 % 40 == 0 {
trace!("import datafiles {} {}", c1, channel);
dbc.query("commit", &[]).await.errconv()?;
dbc.query("begin", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
dbc.query("begin", &[]).await.err_conv()?;
}
if false && c1 >= 30 {
break;
}
}
dbc.query("commit", &[]).await.errconv()?;
dbc.query("commit", &[]).await.err_conv()?;
Ok(())
}
@@ -744,7 +744,7 @@ impl ChannelDatafileDescSink for DatafileDbWriter {
&((k.timebin() + 1) as i64 * k.binsize() as i64),
&serde_json::to_value(k)?,
]
).await.errconv()?;
).await.err_conv()?;
*c1.write()? += 1;
Ok(())
})

76
dbconn/src/search.rs
View File

@@ -2,8 +2,10 @@ use crate::{create_connection, ErrConv};
use err::Error;
use netpod::{
ChannelArchiver, ChannelSearchQuery, ChannelSearchResult, ChannelSearchSingleResult, Database, NodeConfigCached,
ScalarType, ScyllaConfig, Shape,
};
use serde_json::Value as JsVal;
use std::sync::Arc;
pub async fn search_channel_databuffer(
query: ChannelSearchQuery,
@@ -34,7 +36,7 @@ pub async fn search_channel_databuffer(
&[&query.name_regex, &query.source_regex, &query.description_regex, &"asc"],
)
.await
.errconv()?;
.err_conv()?;
let mut res = vec![];
for row in rows {
let shapedb: Option<serde_json::Value> = row.get(4);
@@ -64,6 +66,7 @@ pub async fn search_channel_databuffer(
let k = ChannelSearchSingleResult {
backend: row.get(7),
name: row.get(1),
series: row.get::<_, i64>(0) as u64,
source: row.get(2),
ty,
shape: shape,
@@ -77,6 +80,65 @@ pub async fn search_channel_databuffer(
Ok(ret)
}
pub async fn search_channel_scylla(
query: ChannelSearchQuery,
_scyconf: &ScyllaConfig,
pgconf: &Database,
) -> Result<ChannelSearchResult, Error> {
let empty = if !query.name_regex.is_empty() {
false
} else if !query.source_regex.is_empty() {
false
} else if !query.description_regex.is_empty() {
false
} else {
true
};
if empty {
let ret = ChannelSearchResult { channels: vec![] };
return Ok(ret);
}
let sql = format!(concat!(
"select",
" series, facility, channel, scalar_type, shape_dims",
" from series_by_channel",
" where channel like $1",
));
let u = {
let d = &pgconf;
format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name)
};
let (pgclient, pgconn) = tokio_postgres::connect(&u, tokio_postgres::NoTls).await.err_conv()?;
// TODO use common connection/pool:
tokio::spawn(pgconn);
let pgclient = Arc::new(pgclient);
let rows = pgclient.query(sql.as_str(), &[&query.name_regex]).await.err_conv()?;
let mut res = vec![];
for row in rows {
let series = row.get::<_, i64>(0) as u64;
let facility: String = row.get(1);
let channel: String = row.get(2);
let a: i32 = row.get(3);
let scalar_type = ScalarType::from_scylla_i32(a)?;
let a: Vec<i32> = row.get(4);
let shape = Shape::from_scylla_shape_dims(&a)?;
let k = ChannelSearchSingleResult {
backend: facility,
name: channel,
series,
source: "".into(),
ty: scalar_type.to_variant_str().into(),
shape: shape.to_scylla_vec().into_iter().map(|x| x as u32).collect(),
unit: "".into(),
description: "".into(),
is_api_0: None,
};
res.push(k);
}
let ret = ChannelSearchResult { channels: res };
Ok(ret)
}
pub async fn search_channel_archeng(
query: ChannelSearchQuery,
backend: String,
@@ -106,7 +168,7 @@ pub async fn search_channel_archeng(
" limit 100"
));
let cl = create_connection(database).await?;
let rows = cl.query(sql.as_str(), &[&query.name_regex]).await.errconv()?;
let rows = cl.query(sql.as_str(), &[&query.name_regex]).await.err_conv()?;
let mut res = vec![];
for row in rows {
let name: String = row.get(0);
@@ -174,6 +236,8 @@ pub async fn search_channel_archeng(
let k = ChannelSearchSingleResult {
backend: backend.clone(),
name,
// TODO provide a unique id also within this backend:
series: 0,
source: String::new(),
ty: st.into(),
shape,
@@ -191,9 +255,11 @@ pub async fn search_channel(
query: ChannelSearchQuery,
node_config: &NodeConfigCached,
) -> Result<ChannelSearchResult, Error> {
let database = &node_config.node_config.cluster.database;
if let Some(conf) = node_config.node.channel_archiver.as_ref() {
search_channel_archeng(query, node_config.node_config.cluster.backend.clone(), conf, database).await
let pgconf = &node_config.node_config.cluster.database;
if let Some(scyconf) = node_config.node_config.cluster.scylla.as_ref() {
search_channel_scylla(query, scyconf, pgconf).await
} else if let Some(conf) = node_config.node.channel_archiver.as_ref() {
search_channel_archeng(query, node_config.node_config.cluster.backend.clone(), conf, pgconf).await
} else if let Some(_conf) = node_config.node.archiver_appliance.as_ref() {
// TODO
err::todoval()