daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restructure config between the kinds of backends

Browse Source
This commit is contained in:
Dominik Werder
2022-02-18 19:24:14 +01:00
parent b7aaad7a7b
commit 96fa8b5b09
20 changed files with 195 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dbconn/src/scan.rs
+36 -6
View File
@@ -269,7 +269,14 @@ impl Stream for UpdatedDbWithChannelNamesStream {
msg: format!("Got ident {:?}", pself.ident),
count: 43,
};
let s = FindChannelNamesFromConfigReadDir::new(&pself.node_config.node.data_base_path);
let base_path = &pself
.node_config
.node
.sf_databuffer
.as_ref()
.ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
.data_base_path;
let s = FindChannelNamesFromConfigReadDir::new(base_path);
*pself.find = Some(s);
Ready(Some(Ok(ret)))
}
@@ -347,7 +354,13 @@ pub async fn update_db_with_channel_names(
dbc.query("begin", &[]).await.errconv()?;
let dbc = Arc::new(dbc);
let tx = Arc::new(tx);
find_channel_names_from_config(&node_config.node.data_base_path, |ch| {
let base_path = &node_config
.node
.sf_databuffer
.as_ref()
.ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
.data_base_path;
find_channel_names_from_config(base_path, |ch| {
let ch = ch.to_owned();
let dbc = dbc.clone();
let c1 = c1.clone();
@@ -407,7 +420,14 @@ pub async fn update_db_with_channel_names(
pub fn update_db_with_channel_names_3<'a>(
node_config: &'a NodeConfigCached,
) -> impl Stream<Item = Result<UpdatedDbWithChannelNames, Error>> + 'static {
futures_util::future::ready(node_config.node.data_base_path.clone())
let base_path = &node_config
.node
.sf_databuffer
.as_ref()
.ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))
.unwrap()
.data_base_path;
futures_util::future::ready(base_path.clone())
.then(|path| tokio::fs::read_dir(path))
.map(Result::unwrap)
.map(|rd| {
@@ -552,9 +572,13 @@ pub async fn update_db_with_channel_config(
count_inserted: &mut usize,
count_updated: &mut usize,
) -> Result<UpdateChannelConfigResult, Error> {
let path = node_config
let base_path = &node_config
.node
.data_base_path
.sf_databuffer
.as_ref()
.ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
.data_base_path;
let path = base_path
.join("config")
.join(channel)
.join("latest")
@@ -806,6 +830,12 @@ pub async fn update_db_with_channel_datafiles(
channel: &str,
dbc: Arc<Client>,
) -> Result<(), Error> {
let base_path = &node_config
.node
.sf_databuffer
.as_ref()
.ok_or_else(|| Error::with_msg(format!("missing sf databuffer config in node")))?
.data_base_path;
let writer = DatafileDbWriter {
node_id: node_disk_ident.rowid(),
channel_id: channel_id,
@@ -814,7 +844,7 @@ pub async fn update_db_with_channel_datafiles(
};
let mut n_nothing = 0;
for ks in &[2, 3, 4] {
match find_channel_datafiles_in_ks(&node_config.node.data_base_path, ks_prefix, *ks, channel, &writer).await {
match find_channel_datafiles_in_ks(base_path, ks_prefix, *ks, channel, &writer).await {
/*Err(Error::ChannelDatadirNotFound { .. }) => {
n_nothing += 1;
}*/
dbconn/src/search.rs
+8 -4
View File
@@ -1,6 +1,8 @@
use crate::{create_connection, ErrConv};
use err::Error;
use netpod::{ChannelArchiver, ChannelSearchQuery, ChannelSearchResult, ChannelSearchSingleResult, NodeConfigCached};
use netpod::{
ChannelArchiver, ChannelSearchQuery, ChannelSearchResult, ChannelSearchSingleResult, Database, NodeConfigCached,
};
use serde_json::Value as JsVal;
pub async fn search_channel_databuffer(
@@ -78,7 +80,8 @@ pub async fn search_channel_databuffer(
pub async fn search_channel_archeng(
query: ChannelSearchQuery,
backend: String,
conf: &ChannelArchiver,
_conf: &ChannelArchiver,
database: &Database,
) -> Result<ChannelSearchResult, Error> {
// Channel archiver provides only channel name. Also, search criteria are currently ANDed.
// Therefore search only if user only provides a name criterion.
@@ -102,7 +105,7 @@ pub async fn search_channel_archeng(
" order by c.name",
" limit 100"
));
let cl = create_connection(&conf.database).await?;
let cl = create_connection(database).await?;
let rows = cl.query(sql.as_str(), &[&query.name_regex]).await.errconv()?;
let mut res = vec![];
for row in rows {
@@ -188,8 +191,9 @@ pub async fn search_channel(
query: ChannelSearchQuery,
node_config: &NodeConfigCached,
) -> Result<ChannelSearchResult, Error> {
let database = &node_config.node_config.cluster.database;
if let Some(conf) = node_config.node.channel_archiver.as_ref() {
search_channel_archeng(query, node_config.node.backend.clone(), conf).await
search_channel_archeng(query, node_config.node.backend.clone(), conf, database).await
} else if let Some(_conf) = node_config.node.archiver_appliance.as_ref() {
// TODO
err::todoval()