daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Publish metrics also for metricbeat

Browse Source
This commit is contained in:
Dominik Werder
2023-11-28 15:55:43 +01:00
parent 3ae555565a
commit 71fa333f75
6 changed files with 82 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
daqingest/src/daemon.rs
View File

@@ -103,9 +103,10 @@ impl Daemon {
// Insert queue hook
// let query_item_rx = inserthook::active_channel_insert_hook(query_item_rx);
let local_epics_hostname = ingest_linux::net::local_hostname();
let conn_set_ctrl = CaConnSet::start(
ingest_opts.backend().into(),
ingest_opts.local_epics_hostname(),
local_epics_hostname,
query_item_tx,
channel_info_query_tx,
ingest_opts.clone(),
@@ -557,6 +558,7 @@ static SIGTERM: AtomicUsize = AtomicUsize::new(0);
static SHUTDOWN_SENT: AtomicUsize = AtomicUsize::new(0);
fn handler_sigint(_a: libc::c_int, _b: *const libc::siginfo_t, _c: *const libc::c_void) {
std::process::exit(13);
SIGINT.store(1, atomic::Ordering::Release);
let _ = ingest_linux::signal::unset_signal_handler(libc::SIGINT);
}

5
netfetch/src/conf.rs
View File

@@ -31,7 +31,6 @@ pub struct CaIngestOpts {
insert_worker_concurrency: Option<usize>,
insert_scylla_sessions: Option<usize>,
insert_item_queue_cap: Option<usize>,
local_epics_hostname: Option<String>,
store_workers_rate: Option<u64>,
insert_frac: Option<u64>,
use_rate_limit_queue: Option<bool>,
@@ -95,10 +94,6 @@ impl CaIngestOpts {
self.insert_item_queue_cap.unwrap_or(80000)
}
pub fn local_epics_hostname(&self) -> String {
self.local_epics_hostname.clone().unwrap_or_else(local_hostname)
}
pub fn store_workers_rate(&self) -> u64 {
self.store_workers_rate.unwrap_or(5000)
}

65
netfetch/src/metrics.rs
View File

@@ -72,6 +72,7 @@ impl IntoResponse for CustomErrorResponse {
}
}
#[derive(Clone)]
pub struct StatsSet {
daemon: Arc<DaemonStats>,
ca_conn_set: Arc<CaConnSetStats>,
@@ -230,6 +231,26 @@ impl DaemonComm {
}
}
fn metricbeat(stats_set: &StatsSet) -> axum::Json<serde_json::Value> {
let mut map = serde_json::Map::new();
map.insert("daemon".to_string(), stats_set.daemon.json());
map.insert("insert_worker_stats".to_string(), stats_set.insert_worker_stats.json());
let mut ret = serde_json::Map::new();
ret.insert("daqingest".to_string(), serde_json::Value::Object(map));
axum::Json(serde_json::Value::Object(ret))
}
fn metrics(stats_set: &StatsSet) -> String {
let s1 = stats_set.daemon.prometheus();
let s2 = stats_set.ca_conn_set.prometheus();
let s3 = stats_set.insert_worker_stats.prometheus();
let s4 = stats_set.ca_conn.prometheus();
let s5 = stats_set.series_by_channel_stats.prometheus();
let s6 = stats_set.ca_proto.prometheus();
let s7 = stats_set.ioc_finder_stats.prometheus();
[s1, s2, s3, s4, s5, s6, s7].join("")
}
fn make_routes(dcom: Arc<DaemonComm>, connset_cmd_tx: Sender<CaConnSetEvent>, stats_set: StatsSet) -> axum::Router {
use axum::extract;
use axum::routing::get;
@@ -255,37 +276,29 @@ fn make_routes(dcom: Arc<DaemonComm>, connset_cmd_tx: Sender<CaConnSetEvent>, st
.route(
"/metrics",
get({
//
|| async move {
let s1 = stats_set.daemon.prometheus();
let s2 = stats_set.ca_conn_set.prometheus();
let s3 = stats_set.insert_worker_stats.prometheus();
let s4 = stats_set.ca_conn.prometheus();
let s5 = stats_set.series_by_channel_stats.prometheus();
let s6 = stats_set.ca_proto.prometheus();
let s7 = stats_set.ioc_finder_stats.prometheus();
[s1, s2, s3, s4, s5, s6, s7].join("")
}
let stats_set = stats_set.clone();
|| async move { metrics(&stats_set) }
}),
)
.route(
"/daqingest/metrics",
get({
let stats_set = stats_set.clone();
|| async move { metrics(&stats_set) }
}),
)
.route(
"/daqingest/metricbeat",
get({
let stats_set = stats_set.clone();
|| async move { metricbeat(&stats_set) }
}),
)
.route(
"/metricbeat",
get({
//
|| async move {
axum::Json(serde_json::json!({
"v1": 42_u32,
"o1": {
"v2": 56,
"o2": {
"v3": "test",
},
},
"o5": {
"v6": 89,
},
}))
}
let stats_set = stats_set.clone();
|| async move { metricbeat(&stats_set) }
}),
)
.route(

32
stats_proc/src/stats_proc.rs
View File

@@ -152,6 +152,36 @@ fn stats_struct_impl(st: &StatsStructDef) -> String {
"
)
};
let fn_json = {
let mut buf = String::new();
for x in &st.counters {
buf.push_str(&format!(
"ret.insert(\"{x}\".to_string(), Value::Number(Number::from(self.{x}.load())));\n"
));
}
for x in &st.values {
buf.push_str(&format!(
"ret.insert(\"{x}\".to_string(), Value::Number(Number::from(self.{x}.load())));\n"
));
}
for x in &st.histolog2s {
buf.push_str(&format!("let v = self.{x}.to_json(\"__dummyname__\");\n"));
buf.push_str(&format!("ret.insert(\"{x}\".to_string(), v);\n"));
}
format!(
"
pub fn json(&self) -> stats_types::serde_json::Value {{
use serde_json::Map;
use serde_json::Number;
use serde_json::Value;
use stats_types::serde_json;
let mut ret = Map::new();
// {buf}
Value::Object(ret)
}}
"
)
};
let fn_snapshot = {
let mut init_counters = String::new();
for x in &st.counters {
@@ -193,6 +223,8 @@ impl {name} {{
{fn_prometheus}
{fn_json}
{fn_snapshot}
}}

3
stats_types/Cargo.toml
View File

@@ -7,4 +7,5 @@ edition = "2021"
[lib]
path = "src/stats_types.rs"
#[dependencies]
[dependencies]
serde_json = "1"

6
stats_types/src/stats_types.rs
View File

@@ -1,3 +1,5 @@
pub use serde_json;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering::AcqRel;
use std::sync::atomic::Ordering::Acquire;
@@ -192,6 +194,10 @@ impl HistoLog2 {
ret.push_str("\n");
ret
}
pub fn to_json(&self, _name: &str) -> serde_json::Value {
serde_json::Value::Null
}
}
#[test]