daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Improve channel status return

Browse Source
This commit is contained in:
Dominik Werder
2025-02-18 15:59:25 +01:00
parent 9a81e25625
commit cc251dc433
6 changed files with 234 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
netfetch/src/ca/connset.rs
View File

@@ -20,7 +20,6 @@ use conn::ChannelStateInfo;
use conn::ChannelStatusPartial;
use conn::ConnCommand;
use conn::ConnCommandResult;
use core::fmt;
use dbpg::seriesbychannel::BoxedSend;
use dbpg::seriesbychannel::CanSendChannelInfoResult;
use dbpg::seriesbychannel::ChannelInfoQuery;
@@ -55,6 +54,7 @@ use stats::CaProtoStats;
use stats::IocFinderStats;
use std::collections::BTreeMap;
use std::collections::VecDeque;
use std::fmt;
use std::net::SocketAddr;
use std::net::SocketAddrV4;
use std::pin::Pin;
@@ -584,13 +584,7 @@ impl CaConnSet {
fn handle_add_channel_new(cmd: ChannelAdd, ress: StateTransRes) -> Result<(), Error> {
{
let item = ChannelState {
value: ChannelStateValue::Active(ActiveChannelState::WaitForStatusSeriesId {
since: SystemTime::now(),
}),
config: cmd.ch_cfg.clone(),
touched: 1,
};
let item = ChannelState::new_wait_for_cssid(&cmd.ch_cfg);
*ress.chst = item;
}
{

27
netfetch/src/ca/statemap.rs
View File

@@ -188,6 +188,7 @@ pub struct ChannelState {
pub value: ChannelStateValue,
pub config: ChannelConfig,
pub touched: u8,
config_file_basename: String,
}
impl ChannelState {
@@ -199,6 +200,26 @@ impl ChannelState {
false
}
}
pub fn new_dummy() -> Self {
Self {
value: ChannelStateValue::InitDummy,
config: ChannelConfig::dummy(),
touched: 0,
config_file_basename: String::new(),
}
}
pub fn new_wait_for_cssid(ch_cfg: &crate::conf::ChannelConfig) -> Self {
Self {
value: ChannelStateValue::Active(ActiveChannelState::WaitForStatusSeriesId {
since: SystemTime::now(),
}),
config: ch_cfg.clone(),
touched: 1,
config_file_basename: ch_cfg.config_file_basename().into(),
}
}
}
#[derive(Debug, Serialize)]
@@ -230,11 +251,7 @@ impl ChannelStateMap {
pub fn get_mut_or_dummy_init(&mut self, k: &ChannelName) -> &mut ChannelState {
if !self.map.contains_key(k) {
let dummy = ChannelState {
value: ChannelStateValue::InitDummy,
config: ChannelConfig::dummy(),
touched: 0,
};
let dummy = ChannelState::new_dummy();
self.map.insert(k.clone(), dummy);
}
self.map.get_mut(k).unwrap()

135
netfetch/src/conf.rs
View File

@@ -43,6 +43,7 @@ pub struct CaIngestOpts {
pub test_bsread_addr: Option<String>,
#[serde(default)]
scylla_disable: bool,
#[serde(default)]
scylla_ignore_writes: bool,
}
@@ -188,6 +189,7 @@ fn test_duration_parse() {
}
async fn parse_channel_config_txt(fname: &Path) -> Result<ChannelsConfig, Error> {
let basename = fname.file_stem().unwrap().to_str().unwrap();
let re_p = Regex::new("--------------------------").unwrap();
let re_n = Regex::new("--------------------------").unwrap();
let mut file = OpenOptions::new().read(true).open(fname).await?;
@@ -218,6 +220,7 @@ async fn parse_channel_config_txt(fname: &Path) -> Result<ChannelsConfig, Error>
is_polled: false,
timestamp: ChannelTimestamp::Archiver,
},
config_file_basename: basename.to_string(),
};
conf.channels.push(item);
}
@@ -272,11 +275,12 @@ async fn parse_config_dir(dir: &Path) -> Result<ChannelsConfig, Error> {
let fnp = e.path();
let fns = fnp.to_str().unwrap();
if fns.ends_with(".yml") || fns.ends_with(".yaml") {
let basename = fnp.file_stem().unwrap().to_str().unwrap();
let buf = tokio::fs::read(e.path()).await?;
let conf: BTreeMap<String, ChannelConfigParse> =
serde_yaml::from_slice(&buf).map_err(Error::from_string)?;
info!("parsed {} channels from {}", conf.len(), fns);
ret.push_from_parsed(&conf);
ret.push_from_parsed(&conf, basename);
} else {
debug!("ignore channel config file {:?}", e.path());
}
@@ -299,23 +303,31 @@ impl ChannelTimestamp {
fn default_config() -> Self {
Self::Archiver
}
fn is_default(&self) -> bool {
if let ChannelTimestamp::Archiver = self {
true
} else {
false
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Deserialize)]
pub struct IngestConfigArchiving {
#[serde(default = "bool_true")]
#[serde(with = "serde_replication_bool")]
replication: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
#[serde(with = "serde_option_channel_read_config")]
short_term: Option<ChannelReadConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
#[serde(with = "serde_option_channel_read_config")]
medium_term: Option<ChannelReadConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
#[serde(with = "serde_option_channel_read_config")]
long_term: Option<ChannelReadConfig>,
#[serde(default, skip_serializing_if = "bool_is_false")]
#[serde(default)]
is_polled: bool,
#[serde(default = "ChannelTimestamp::default_config")]
timestamp: ChannelTimestamp,
@@ -343,6 +355,71 @@ fn bool_true() -> bool {
true
}
mod serde_ingest_config_archiving {
use super::ChannelReadConfigApiFormat;
use super::IngestConfigArchiving;
use serde::de;
use serde::ser;
use serde::ser::SerializeMap;
use serde::Deserializer;
use serde::Serializer;
use std::fmt;
impl ser::Serialize for IngestConfigArchiving {
fn serialize<S>(&self, ser: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = ser.serialize_map(None)?;
// ser.is_human_readable()
if !self.replication {
map.serialize_entry("replication", &self.replication)?;
}
if let Some(v) = self.short_term.as_ref() {
map.serialize_entry("short_term", &ChannelReadConfigApiFormat(&v))?;
}
if let Some(v) = self.medium_term.as_ref() {
map.serialize_entry("medium_term", &ChannelReadConfigApiFormat(&v))?;
}
if let Some(v) = self.long_term.as_ref() {
map.serialize_entry("long_term", &ChannelReadConfigApiFormat(&v))?;
}
let anymon = [&self.short_term, &self.medium_term, &self.long_term]
.into_iter()
.map(|c| c.as_ref().map_or(false, |x| x.is_monitor()))
.fold(false, |a, x| a || x);
if anymon && self.is_polled || !anymon && !self.is_polled {
map.serialize_entry("is_polled", &self.is_polled)?;
}
if !self.timestamp.is_default() {
map.serialize_entry("timestamp", &self.timestamp)?;
}
map.end()
}
}
}
struct ChannelReadConfigApiFormat<'a>(&'a ChannelReadConfig);
#[allow(non_snake_case)]
mod serde_ChannelReadConfigApiFormat {
use super::ChannelReadConfig;
use super::ChannelReadConfigApiFormat;
use serde::ser;
impl<'a> ser::Serialize for ChannelReadConfigApiFormat<'a> {
fn serialize<S>(&self, ser: S) -> Result<S::Ok, S::Error>
where
S: ser::Serializer,
{
match &self.0 {
ChannelReadConfig::Monitor => ser.serialize_str("Monitor"),
ChannelReadConfig::Poll(n) => ser.serialize_u32(n.as_secs() as u32),
}
}
}
}
mod serde_replication_bool {
use serde::de;
use serde::Deserializer;
@@ -489,6 +566,16 @@ pub enum ChannelReadConfig {
Poll(Duration),
}
impl ChannelReadConfig {
pub fn is_monitor(&self) -> bool {
if let Self::Monitor = self {
true
} else {
false
}
}
}
#[test]
fn test_channel_config_00() {
let inp = r###"
@@ -546,38 +633,27 @@ impl ChannelsConfig {
&self.channels
}
fn push_from_parsed(&mut self, rhs: &BTreeMap<String, ChannelConfigParse>) {
fn push_from_parsed(&mut self, rhs: &BTreeMap<String, ChannelConfigParse>, config_file_basename: &str) {
for (k, v) in rhs.iter() {
let item = ChannelConfig {
name: k.into(),
arch: v.archiving_configuration.clone(),
config_file_basename: config_file_basename.into(),
};
self.channels.push(item);
}
}
}
impl From<BTreeMap<String, ChannelConfigParse>> for ChannelsConfig {
fn from(value: BTreeMap<String, ChannelConfigParse>) -> Self {
let channels = value
.into_iter()
.map(|(k, v)| ChannelConfig {
name: k,
arch: v.archiving_configuration,
})
.collect();
ChannelsConfig { channels }
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct ChannelConfig {
name: String,
arch: IngestConfigArchiving,
config_file_basename: String,
}
impl ChannelConfig {
pub fn st_monitor<S: Into<String>>(name: S) -> Self {
pub fn st_monitor<S: Into<String>>(name: S, config_file_basename: &str) -> Self {
Self {
name: name.into(),
arch: IngestConfigArchiving {
@@ -588,6 +664,7 @@ impl ChannelConfig {
is_polled: false,
timestamp: ChannelTimestamp::Archiver,
},
config_file_basename: config_file_basename.into(),
}
}
@@ -595,6 +672,10 @@ impl ChannelConfig {
&self.name
}
pub fn config_file_basename(&self) -> &str {
&self.config_file_basename
}
pub fn is_polled(&self) -> bool {
self.arch.is_polled
}
@@ -678,6 +759,18 @@ impl ChannelConfig {
Self {
name: String::from("dummy"),
arch: IngestConfigArchiving::dummy(),
config_file_basename: String::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct ChannelConfigForStatesApi {
arch: IngestConfigArchiving,
}
impl From<ChannelConfig> for ChannelConfigForStatesApi {
fn from(value: ChannelConfig) -> Self {
Self { arch: value.arch }
}
}

2
netfetch/src/metrics.rs
View File

@@ -215,7 +215,7 @@ async fn find_channel(
async fn channel_add_inner(params: HashMap<String, String>, dcom: Arc<DaemonComm>) -> Result<(), Error> {
if let Some(name) = params.get("name") {
// let ch = crate::daemon_common::Channel::new(name.into());
let ch_cfg = ChannelConfig::st_monitor(name);
let ch_cfg = ChannelConfig::st_monitor(name, "api");
let (tx, rx) = async_channel::bounded(1);
let ev = DaemonEvent::ChannelAdd(ch_cfg, tx);
dcom.tx.send(ev).await?;

20
netfetch/src/metrics/status.rs
View File

@@ -2,6 +2,7 @@ use crate::ca::connset::CaConnSetEvent;
use crate::ca::connset::ChannelStatusesRequest;
use crate::ca::connset::ConnSetCmd;
use crate::conf::ChannelConfig;
use crate::conf::ChannelConfigForStatesApi;
use async_channel::Sender;
use chrono::DateTime;
use chrono::Utc;
@@ -62,7 +63,7 @@ impl StorageUsage {
struct ChannelState {
ioc_address: Option<SocketAddr>,
connection: ConnectionState,
archiving_configuration: ChannelConfig,
archiving_configuration: ChannelConfigForStatesApi,
recv_count: u64,
recv_bytes: u64,
#[serde(with = "humantime_serde", skip_serializing_if = "system_time_epoch")]
@@ -87,9 +88,10 @@ impl ChannelState {
fn connecting_addr(config: ChannelConfig, ioc_address: Option<SocketAddr>, connst: ConnectionState) -> Self {
Self {
private: StatePrivate::default(config.config_file_basename()),
ioc_address,
connection: connst,
archiving_configuration: config,
archiving_configuration: config.into(),
recv_count: 0,
recv_bytes: 0,
recv_last: SystemTime::UNIX_EPOCH,
@@ -98,13 +100,13 @@ impl ChannelState {
write_lt_last: SystemTime::UNIX_EPOCH,
updated: SystemTime::UNIX_EPOCH,
pong_last: None,
private: StatePrivate::default(),
}
}
fn with_chst(config: ChannelConfig, chst: crate::ca::conn::ChannelStateInfo) -> Self {
let private = StatePrivate {
status_emit_count: chst.status_emit_count,
config_file_basename: config.config_file_basename().into(),
};
let connst = {
use crate::ca::conn::ChannelConnectedInfo::*;
@@ -120,7 +122,7 @@ impl ChannelState {
connection: connst,
// TODO config is stored in two places
// conf: chst.conf,
archiving_configuration: config,
archiving_configuration: config.into(),
recv_count: chst.recv_count.unwrap_or(0),
recv_bytes: chst.recv_bytes.unwrap_or(0),
recv_last: chst.recv_last,
@@ -137,11 +139,15 @@ impl ChannelState {
#[derive(Debug, Serialize)]
struct StatePrivate {
status_emit_count: u64,
config_file_basename: String,
}
impl Default for StatePrivate {
fn default() -> Self {
Self { status_emit_count: 0 }
impl StatePrivate {
fn default(config_file_basename: &str) -> Self {
Self {
status_emit_count: 0,
config_file_basename: config_file_basename.into(),
}
}
}