daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor usage of common error type more

Browse Source
This commit is contained in:
Dominik Werder
2021-12-08 13:20:07 +01:00
parent c39af81097
commit 3c64eafd14
56 changed files with 751 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
dbconn/src/lib.rs
View File

@@ -1,14 +1,36 @@
pub mod scan;
pub mod search;
pub mod pg {
pub use tokio_postgres::{Client, Error};
}
use err::Error;
use netpod::log::*;
use netpod::{Channel, Database, NodeConfigCached};
use std::time::Duration;
use tokio_postgres::{Client, NoTls};
pub mod scan;
pub mod search;
trait ErrConv<T> {
fn errconv(self) -> Result<T, Error>;
}
pub mod pg {
pub use tokio_postgres::Client;
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
fn errconv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg(e.to_string())),
}
}
}
impl<T, A> ErrConv<T> for Result<T, async_channel::SendError<A>> {
fn errconv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg(e.to_string())),
}
}
}
pub async fn delay_us(mu: u64) {
@@ -26,7 +48,7 @@ pub async fn delay_io_medium() {
pub async fn create_connection(db_config: &Database) -> Result<Client, Error> {
let d = db_config;
let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, 5432, d.name);
let (cl, conn) = tokio_postgres::connect(&uri, NoTls).await?;
let (cl, conn) = tokio_postgres::connect(&uri, NoTls).await.errconv()?;
// TODO monitor connection drop.
let _cjh = tokio::spawn(async move {
if let Err(e) = conn.await {
@@ -41,7 +63,8 @@ pub async fn channel_exists(channel: &Channel, node_config: &NodeConfigCached) -
let cl = create_connection(&node_config.node_config.cluster.database).await?;
let rows = cl
.query("select rowid from channels where name = $1::text", &[&channel.name])
.await?;
.await
.errconv()?;
debug!("channel_exists {} rows", rows.len());
for row in rows {
debug!(
@@ -61,7 +84,8 @@ pub async fn database_size(node_config: &NodeConfigCached) -> Result<u64, Error>
"select pg_database_size($1::text)",
&[&node_config.node_config.cluster.database.name],
)
.await?;
.await
.errconv()?;
if rows.len() == 0 {
Err(Error::with_msg("could not get database size"))?;
}
@@ -87,7 +111,7 @@ pub async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, E
);
let sql = sql.as_str();
let cl = create_connection(&node_config.node_config.cluster.database).await?;
let rows = cl.query(sql, &[]).await?;
let rows = cl.query(sql, &[]).await.errconv()?;
let mut sizes = TableSizes { sizes: vec![] };
sizes.sizes.push((format!("table"), format!("size")));
for row in rows {
@@ -99,7 +123,7 @@ pub async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, E
pub async fn random_channel(node_config: &NodeConfigCached) -> Result<String, Error> {
let sql = "select name from channels order by rowid limit 1 offset (random() * (select count(rowid) from channels))::bigint";
let cl = create_connection(&node_config.node_config.cluster.database).await?;
let rows = cl.query(sql, &[]).await?;
let rows = cl.query(sql, &[]).await.errconv()?;
if rows.len() == 0 {
Err(Error::with_msg("can not get random channel"))?;
}
@@ -112,11 +136,12 @@ pub async fn insert_channel(name: String, facility: i64, dbc: &Client) -> Result
"select count(rowid) from channels where facility = $1 and name = $2",
&[&facility, &name],
)
.await?;
.await
.errconv()?;
if rows[0].get::<_, i64>(0) == 0 {
let sql =
concat!("insert into channels (facility, name) values ($1, $2) on conflict (facility, name) do nothing");
dbc.query(sql, &[&facility, &name]).await?;
dbc.query(sql, &[&facility, &name]).await.errconv()?;
}
Ok(())
}

87
dbconn/src/scan.rs
View File

@@ -6,6 +6,7 @@ use futures_core::Stream;
use futures_util::{pin_mut, FutureExt, StreamExt};
use netpod::log::*;
use netpod::{Database, NodeConfigCached};
use parse::channelconfig::NErr;
use pin_project::pin_project;
use serde::{Deserialize, Serialize};
use std::future::Future;
@@ -19,6 +20,8 @@ use std::task::{Context, Poll};
use tokio::fs::{DirEntry, ReadDir};
use tokio_postgres::Client;
use crate::ErrConv;
#[derive(Debug, Serialize, Deserialize)]
pub struct NodeDiskIdent {
pub rowid: i64,
@@ -45,7 +48,8 @@ pub async fn get_node_disk_ident(node_config: &NodeConfigCached, dbc: &Client) -
let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
let rows = dbc
.query(sql, &[&node_config.node.backend, &node_config.node.host])
.await?;
.await
.errconv()?;
if rows.len() != 1 {
return Err(Error::with_msg(format!(
"get_node can't find unique entry for {} {}",
@@ -68,7 +72,8 @@ pub async fn get_node_disk_ident_2(
let sql = "select nodes.rowid, facility, split, hostname from nodes, facilities where facilities.name = $1 and facility = facilities.rowid and hostname = $2";
let rows = dbc
.query(sql, &[&node_config.node.backend, &node_config.node.host])
.await?;
.await
.errconv()?;
if rows.len() != 1 {
return Err(Error::with_msg(format!(
"get_node can't find unique entry for {} {}",
@@ -315,15 +320,16 @@ impl Stream for UpdatedDbWithChannelNamesStream {
async fn update_db_with_channel_name_list(list: Vec<String>, backend: i64, dbc: &Client) -> Result<(), Error> {
crate::delay_io_short().await;
dbc.query("begin", &[]).await?;
dbc.query("begin", &[]).await.errconv()?;
for ch in list {
dbc.query(
"insert into channels (facility, name) values ($1, $2) on conflict do nothing",
&[&backend, &ch],
)
.await?;
.await
.errconv()?;
}
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
Ok(())
}
@@ -338,7 +344,7 @@ pub async fn update_db_with_channel_names(
let dbc = crate::create_connection(&db_config).await?;
let node_disk_ident = get_node_disk_ident(&node_config, &dbc).await?;
let c1 = Arc::new(RwLock::new(0u32));
dbc.query("begin", &[]).await?;
dbc.query("begin", &[]).await.errconv()?;
let dbc = Arc::new(dbc);
let tx = Arc::new(tx);
find_channel_names_from_config(&node_config.node.data_base_path, |ch| {
@@ -353,33 +359,34 @@ pub async fn update_db_with_channel_names(
"insert into channels (facility, name) values ($1, $2) on conflict do nothing",
&[&fac, &ch],
)
.await?;
.await
.errconv()?;
let c2 = {
let mut g = c1.write()?;
*g += 1;
*g
};
if c2 % 200 == 0 {
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
let ret = UpdatedDbWithChannelNames {
msg: format!("current {}", ch),
count: c2,
};
tx.send(Ok(ret)).await?;
tx.send(Ok(ret)).await.errconv()?;
crate::delay_io_medium().await;
dbc.query("begin", &[]).await?;
dbc.query("begin", &[]).await.errconv()?;
}
Ok(())
}
})
.await?;
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
let c2 = *c1.read()?;
let ret = UpdatedDbWithChannelNames {
msg: format!("all done"),
count: c2,
};
tx.send(Ok(ret)).await?;
tx.send(Ok(ret)).await.errconv()?;
Ok::<_, Error>(())
};
let block2 = async move {
@@ -440,15 +447,16 @@ pub async fn update_db_with_all_channel_configs(
"select rowid, facility, name from channels where facility = $1 order by facility, name",
&[&node_disk_ident.facility],
)
.await?;
.await
.errconv()?;
let mut c1 = 0;
dbc.query("begin", &[]).await?;
dbc.query("begin", &[]).await.errconv()?;
let mut count_inserted = 0;
let mut count_updated = 0;
for row in rows {
let rowid: i64 = row.try_get(0)?;
let _facility: i64 = row.try_get(1)?;
let channel: String = row.try_get(2)?;
let rowid: i64 = row.try_get(0).errconv()?;
let _facility: i64 = row.try_get(1).errconv()?;
let channel: String = row.try_get(2).errconv()?;
match update_db_with_channel_config(
node_config,
node_disk_ident,
@@ -471,26 +479,26 @@ pub async fn update_db_with_all_channel_configs(
Ok(UpdateChannelConfigResult::Done) => {
c1 += 1;
if c1 % 200 == 0 {
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
let msg = format!(
"channel no {:6} inserted {:6} updated {:6}",
c1, count_inserted, count_updated
);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
tx.send(Ok(ret)).await?;
dbc.query("begin", &[]).await?;
tx.send(Ok(ret)).await.errconv()?;
dbc.query("begin", &[]).await.errconv()?;
}
crate::delay_io_short().await;
}
}
}
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
let msg = format!(
"ALL DONE channel no {:6} inserted {:6} updated {:6}",
c1, count_inserted, count_updated
);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: c1 };
tx.send(Ok(ret)).await?;
tx.send(Ok(ret)).await.errconv()?;
Ok::<_, Error>(())
}
.then({
@@ -500,7 +508,7 @@ pub async fn update_db_with_all_channel_configs(
Err(e) => {
let msg = format!("Seeing error: {:?}", e);
let ret = UpdatedDbWithAllChannelConfigs { msg, count: 0 };
tx2.send(Ok(ret)).await?;
tx2.send(Ok(ret)).await.errconv()?;
}
}
Ok::<_, Error>(())
@@ -523,7 +531,7 @@ pub async fn update_db_with_all_channel_configs(
pub async fn update_search_cache(node_config: &NodeConfigCached) -> Result<(), Error> {
let dbc = crate::create_connection(&node_config.node_config.cluster.database).await?;
dbc.query("select update_cache()", &[]).await?;
dbc.query("select update_cache()", &[]).await.errconv()?;
Ok(())
}
@@ -564,7 +572,8 @@ pub async fn update_db_with_channel_config(
"select rowid, fileSize, parsedUntil, channel from configs where node = $1 and channel = $2",
&[&node_disk_ident.rowid(), &channel_id],
)
.await?;
.await
.errconv()?;
if rows.len() > 1 {
return Err(Error::with_msg("more than one row"));
}
@@ -579,7 +588,7 @@ pub async fn update_db_with_channel_config(
"insert into configs_history (rowid_original, node, channel, fileSize, parsedUntil, config, tsinsert) ",
"select rowid as rowid_original, node, channel, fileSize, parsedUntil, config, now() from configs where rowid = $1"
);
dbc.query(sql, &[&rowid]).await?;
dbc.query(sql, &[&rowid]).await.errconv()?;
}
//ensure!(meta.len() >= parsed_until as u64, ConfigFileOnDiskShrunk{path});
(Some(rowid), true)
@@ -588,7 +597,7 @@ pub async fn update_db_with_channel_config(
};
if do_parse {
let buf = tokio::fs::read(&path).await?;
let config = parse::channelconfig::parse_config(&buf)?.1;
let config = parse::channelconfig::parse_config(&buf).map_err(NErr::from)?.1;
match config_id {
None => {
dbc.query(
@@ -601,14 +610,15 @@ pub async fn update_db_with_channel_config(
&serde_json::to_value(config)?,
],
)
.await?;
.await
.errconv()?;
*count_inserted += 1;
}
Some(_config_id_2) => {
dbc.query(
"insert into configs (node, channel, fileSize, parsedUntil, config) values ($1, $2, $3, $4, $5) on conflict (node, channel) do update set fileSize = $3, parsedUntil = $4, config = $5",
&[&node_disk_ident.rowid(), &channel_id, &(meta.len() as i64), &(buf.len() as i64), &serde_json::to_value(config)?],
).await?;
).await.errconv()?;
*count_updated += 1;
}
}
@@ -627,25 +637,26 @@ pub async fn update_db_with_all_channel_datafiles(
"select rowid, facility, name from channels where facility = $1 order by facility, name",
&[&node_disk_ident.facility()],
)
.await?;
.await
.errconv()?;
let mut c1 = 0;
dbc.query("begin", &[]).await?;
dbc.query("begin", &[]).await.errconv()?;
for row in rows {
let rowid: i64 = row.try_get(0)?;
let _facility: i64 = row.try_get(1)?;
let channel: String = row.try_get(2)?;
let rowid: i64 = row.try_get(0).errconv()?;
let _facility: i64 = row.try_get(1).errconv()?;
let channel: String = row.try_get(2).errconv()?;
update_db_with_channel_datafiles(node_config, node_disk_ident, ks_prefix, rowid, &channel, dbc.clone()).await?;
c1 += 1;
if c1 % 40 == 0 {
trace!("import datafiles {} {}", c1, channel);
dbc.query("commit", &[]).await?;
dbc.query("begin", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
dbc.query("begin", &[]).await.errconv()?;
}
if false && c1 >= 30 {
break;
}
}
dbc.query("commit", &[]).await?;
dbc.query("commit", &[]).await.errconv()?;
Ok(())
}
@@ -709,7 +720,7 @@ impl ChannelDatafileDescSink for DatafileDbWriter {
&((k.timebin() + 1) as i64 * k.binsize() as i64),
&serde_json::to_value(k)?,
]
).await?;
).await.errconv()?;
*c1.write()? += 1;
Ok(())
})

27
dbconn/src/search.rs
View File

@@ -1,4 +1,4 @@
use crate::create_connection;
use crate::{create_connection, ErrConv};
use err::Error;
use netpod::{ChannelArchiver, ChannelSearchQuery, ChannelSearchResult, ChannelSearchSingleResult, NodeConfigCached};
use serde_json::Value as JsVal;
@@ -18,7 +18,8 @@ pub async fn search_channel_databuffer(
sql.as_str(),
&[&query.name_regex, &query.source_regex, &query.description_regex, &"asc"],
)
.await?;
.await
.errconv()?;
let mut res = vec![];
for row in rows {
let shapedb: Option<serde_json::Value> = row.get(4);
@@ -74,7 +75,7 @@ pub async fn search_channel_archeng(
" limit 100"
));
let cl = create_connection(&conf.database).await?;
let rows = cl.query(sql.as_str(), &[&query.name_regex]).await?;
let rows = cl.query(sql.as_str(), &[&query.name_regex]).await.errconv()?;
let mut res = vec![];
for row in rows {
let name: String = row.get(0);
@@ -105,7 +106,10 @@ pub async fn search_channel_archeng(
if k == "Scalar" {
vec![]
} else {
return Err(Error::with_msg_no_trace(format!("search_channel_archeng can not understand {:?}", config)));
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
}
}
JsVal::Object(k) => match k.get("Wave") {
@@ -114,15 +118,24 @@ pub async fn search_channel_archeng(
vec![k.as_i64().unwrap_or(u32::MAX as i64) as u32]
}
_ => {
return Err(Error::with_msg_no_trace(format!("search_channel_archeng can not understand {:?}", config)));
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
}
},
None => {
return Err(Error::with_msg_no_trace(format!("search_channel_archeng can not understand {:?}", config)));
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
}
},
_ => {
return Err(Error::with_msg_no_trace(format!("search_channel_archeng can not understand {:?}", config)));
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
}
},
None => vec![],