daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restructure config between the kinds of backends

Browse Source
This commit is contained in:
Dominik Werder
2022-02-18 19:24:14 +01:00
parent b7aaad7a7b
commit 96fa8b5b09
20 changed files with 195 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
httpret/src/channelarchiver.rs
View File

@@ -111,7 +111,7 @@ impl ScanIndexFiles {
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_index_files(conf.clone());
let s = archapp_wrap::archapp::archeng::indexfiles::scan_index_files(conf.clone(), node_config.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
@@ -151,7 +151,7 @@ impl ScanChannels {
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_channels(conf.clone());
let s = archapp_wrap::archapp::archeng::indexfiles::scan_channels(node_config.clone(), conf.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
@@ -184,15 +184,9 @@ impl ChannelNames {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let database = &node_config.node_config.cluster.database;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = archeng::configs::ChannelNameStream::new(database.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
@@ -225,6 +219,7 @@ impl ScanConfigs {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let database = &node_config.node_config.cluster.database;
let conf = node_config
.node
.channel_archiver
@@ -233,8 +228,8 @@ impl ScanConfigs {
"this node is not configured as channel archiver",
))?;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = archeng::configs::ConfigStream::new(stream, conf.clone());
let stream = archeng::configs::ChannelNameStream::new(database.clone());
let stream = archeng::configs::ConfigStream::new(stream, node_config.clone(), conf.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
@@ -267,13 +262,7 @@ impl BlockRefStream {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let database = &node_config.node_config.cluster.database;
let range = NanoRange { beg: 0, end: u64::MAX };
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
@@ -284,7 +273,7 @@ impl BlockRefStream {
//name: "ARIDI-PCT:CURRENT".into(),
};
use archapp_wrap::archapp::archeng;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), conf.database.clone()).await?;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), database.clone()).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let s = archeng::blockrefstream::blockref_stream(channel, range, true, ixpath);
@@ -337,13 +326,7 @@ impl BlockStream {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let database = &node_config.node_config.cluster.database;
let range = NanoRange { beg: 0, end: u64::MAX };
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
@@ -354,7 +337,7 @@ impl BlockStream {
name: channel_name,
};
use archapp_wrap::archapp::archeng;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), conf.database.clone()).await?;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), database.clone()).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let s = archeng::blockrefstream::blockref_stream(channel, range.clone(), true, ixpath);

3
httpret/src/httpret.rs
View File

@@ -782,7 +782,8 @@ pub async fn channel_config(req: Request<Body>, node_config: &NodeConfigCached)
//let pairs = get_url_query_pairs(&url);
let q = ChannelConfigQuery::from_url(&url)?;
let conf = if let Some(conf) = &node_config.node.channel_archiver {
archapp_wrap::archapp::archeng::channel_config_from_db(&q, conf).await?
archapp_wrap::archapp::archeng::channel_config_from_db(&q, conf, &node_config.node_config.cluster.database)
.await?
} else if let Some(conf) = &node_config.node.archiver_appliance {
archapp_wrap::channel_config(&q, conf).await?
} else {

2
httpret/src/proxy.rs
View File

@@ -544,7 +544,7 @@ fn get_query_host_for_backend(backend: &str, proxy_config: &ProxyConfig) -> Resu
}
fn get_query_host_for_backend_2(backend: &str, proxy_config: &ProxyConfig) -> Result<String, Error> {
for back in &proxy_config.backends2 {
for back in &proxy_config.backends_pulse_map {
if back.name == backend {
return Ok(back.url.clone());
}

6
httpret/src/pulsemap.rs
View File

@@ -78,10 +78,10 @@ fn timer_channel_names() -> Vec<String> {
async fn datafiles_for_channel(name: String, node_config: &NodeConfigCached) -> Result<Vec<PathBuf>, Error> {
let mut a = vec![];
let n = &node_config.node;
let channel_path = n
let sfc = node_config.node.sf_databuffer.as_ref().unwrap();
let channel_path = sfc
.data_base_path
.join(format!("{}_2", n.ksprefix))
.join(format!("{}_2", sfc.ksprefix))
.join("byTime")
.join(&name);
let mut rd = tokio::fs::read_dir(&channel_path).await?;