From 7edab8f24a12bc39b1404510b0b0e76ee9d8cecc Mon Sep 17 00:00:00 2001 From: Dominik Werder Date: Thu, 10 Apr 2025 11:55:52 +0200 Subject: [PATCH] Add log2 histo --- Cargo.toml | 4 +- src/make_metrics.rs | 123 ++++++++++++++++++----- src/make_metrics/codegen.rs | 188 ++++++++++++------------------------ 3 files changed, 164 insertions(+), 151 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 67ea085..74b913b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mettrics-macros" -version = "0.0.5" +version = "0.0.6" description = "Macros for mettrics crate." authors = ["Dominik Werder "] license = "GPL-3.0-only" @@ -14,3 +14,5 @@ proc-macro2 = "1" syn = { version = "2", features = ["full", "parsing"] } quote = "1" prettyplease = "0.2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/src/make_metrics.rs b/src/make_metrics.rs index 5bd0a40..5cbf2a6 100644 --- a/src/make_metrics.rs +++ b/src/make_metrics.rs @@ -4,9 +4,57 @@ mod resolve; use crate::log::log; use proc_macro2::Span; use proc_macro2::TokenStream; +use std::fmt; use syn::ext::IdentExt; use syn::parse::ParseStream; +#[allow(unused)] +#[derive(Debug)] +pub enum Error { + Deser, +} + +impl fmt::Display for Error { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, fmt) + } +} + +#[allow(non_snake_case)] +mod serde_syn_Path { + use std::fmt; + + pub fn serialize(val: &syn::Path, ser: S) -> Result + where + S: serde::Serializer, + { + let s = quote::quote! { #val }.to_string(); + ser.serialize_str(&s) + } + + struct Vis; + + impl<'a> serde::de::Visitor<'a> for Vis { + type Value = syn::Path; + + fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "expect a path") + } + + fn visit_str(self, inp: &str) -> Result { + let val = syn::parse_str::(inp).expect("a syn parseable path"); + Ok(val) + } + } + + pub fn deserialize<'a, D>(de: D) -> Result + where + D: serde::Deserializer<'a>, + { + de.deserialize_str(Vis) + } +} + #[allow(unused)] fn discard_rest(inp: ParseStream) { inp.step(|c| { @@ -50,7 +98,7 @@ impl syn::parse::Parse for MetricsStructNameItem { } } -#[derive(Debug)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct ComposeAggModItem { input: String, aggtor: String, @@ -70,7 +118,8 @@ impl syn::parse::Parse for ComposeAggModItem { if item.ident.to_string() == "Input" { match item.ty.as_ref() { syn::Type::Path(tp) => { - input = Some(tp.path.get_ident().unwrap().to_string()); + input = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -80,7 +129,8 @@ impl syn::parse::Parse for ComposeAggModItem { } else if item.ident.to_string() == "Aggtor" { match item.ty.as_ref() { syn::Type::Path(tp) => { - aggtor = Some(tp.path.get_ident().unwrap().to_string()); + aggtor = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -90,7 +140,8 @@ impl syn::parse::Parse for ComposeAggModItem { } else if item.ident.to_string() == "Name" { match item.ty.as_ref() { syn::Type::Path(tp) => { - name = Some(tp.path.get_ident().unwrap().to_string()); + name = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -110,15 +161,15 @@ impl syn::parse::Parse for ComposeAggModItem { } } let ret = Self { - input: input.unwrap(), - aggtor: aggtor.unwrap(), - name: name.unwrap(), + input: input.expect("type Input"), + aggtor: aggtor.expect("type Aggtor"), + name: name.expect("type Name"), }; Ok(ret) } } -#[derive(Debug)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct AggregationModItem { struct_name: String, input: String, @@ -140,7 +191,8 @@ impl syn::parse::Parse for AggregationModItem { if item.ident.to_string() == "StructName" { match item.ty.as_ref() { syn::Type::Path(tp) => { - struct_name = Some(tp.path.get_ident().unwrap().to_string()); + struct_name = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -150,7 +202,8 @@ impl syn::parse::Parse for AggregationModItem { } else if item.ident.to_string() == "Input" { match item.ty.as_ref() { syn::Type::Path(tp) => { - input = Some(tp.path.get_ident().unwrap().to_string()); + input = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -177,20 +230,32 @@ impl syn::parse::Parse for AggregationModItem { } } let ret = Self { - struct_name: struct_name.unwrap(), - input: input.unwrap(), + struct_name: struct_name.expect("type StructName"), + input: input.expect("type Input"), compose_agg_mods, }; Ok(ret) } } -#[derive(Debug)] +#[derive(serde::Serialize, serde::Deserialize)] struct ComposeModItem { - input: String, + #[serde(with = "serde_syn_Path")] + input: syn::Path, name: String, } +impl fmt::Debug for ComposeModItem { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let input = &self.input; + let input = quote::quote! { #input }.to_string(); + fmt.debug_struct("ComposeModItem") + .field("input", &input) + .field("name", &self.name) + .finish() + } +} + impl syn::parse::Parse for ComposeModItem { fn parse(inp: ParseStream) -> syn::Result { let mut input = None; @@ -205,7 +270,7 @@ impl syn::parse::Parse for ComposeModItem { if item.ident.to_string() == "Input" { match item.ty.as_ref() { syn::Type::Path(tp) => { - input = Some(tp.path.get_ident().unwrap().to_string()); + input = Some(tp.path.clone()); } _ => { let e = inp.error(format!("expect a path type")); @@ -215,7 +280,8 @@ impl syn::parse::Parse for ComposeModItem { } else if item.ident.to_string() == "Name" { match item.ty.as_ref() { syn::Type::Path(tp) => { - name = Some(tp.path.get_ident().unwrap().to_string()); + name = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -235,17 +301,18 @@ impl syn::parse::Parse for ComposeModItem { } } let ret = Self { - input: input.unwrap(), - name: name.unwrap(), + input: input.expect("type Input"), + name: name.expect("type Name"), }; Ok(ret) } } -#[derive(Debug)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct MetricsModItem { struct_name: String, counter_names: Vec, + histolog2_names: Vec, compose_mods: Vec, } @@ -255,6 +322,7 @@ impl syn::parse::Parse for MetricsModItem { fn parse(inp: ParseStream) -> syn::Result { let mut struct_name = None; let mut counter_names = Vec::new(); + let mut histolog2_names = Vec::new(); let mut compose_mods = Vec::new(); log(&format!("MetricsModItem inp 1 {:?}", inp)); let item = inp.parse::()?; @@ -266,7 +334,8 @@ impl syn::parse::Parse for MetricsModItem { if item.ident.to_string() == "StructName" { match item.ty.as_ref() { syn::Type::Path(tp) => { - struct_name = Some(tp.path.get_ident().unwrap().to_string()); + struct_name = + Some(tp.path.get_ident().expect("path-ident").to_string()); } _ => { let e = inp.error(format!("expect a path type")); @@ -302,8 +371,13 @@ impl syn::parse::Parse for MetricsModItem { let s = var.ident.to_string(); counter_names.push(s); } + } else if idn == "histolog2s" { + for var in vars { + let s = var.ident.to_string(); + histolog2_names.push(s); + } } else { - let e = inp.error(format!("expect enum `values` or `counters`")); + let e = inp.error(format!("expect enum `counters` or `histolog2s`")); return Err(e); } } @@ -315,15 +389,16 @@ impl syn::parse::Parse for MetricsModItem { } } let ret = Self { - struct_name: struct_name.unwrap(), + struct_name: struct_name.expect("type StructName"), counter_names, + histolog2_names, compose_mods, }; Ok(ret) } } -#[derive(Debug)] +#[derive(Debug, serde::Serialize)] pub(crate) struct MetricsDecl { metrics_mods: Vec, agg_mods: Vec, @@ -378,7 +453,7 @@ impl syn::parse::Parse for MetricsDecl { match item.ty.as_ref() { syn::Type::Path(tp) => { let s1 = item.ident.to_string(); - let s2 = tp.path.get_ident().unwrap().to_string(); + let s2 = tp.path.get_ident().expect("path-ident").to_string(); if s1 == "StructName" { log(&format!("have {:?} {:?}", s1, s2)); // struct_name = Some(s2); diff --git a/src/make_metrics/codegen.rs b/src/make_metrics/codegen.rs index 5a70fcc..56edf13 100644 --- a/src/make_metrics/codegen.rs +++ b/src/make_metrics/codegen.rs @@ -1,120 +1,11 @@ use super::AggregationModItem; use super::MetricsDecl; use super::MetricsModItem; -use crate::log::log; use crate::log::log_ts; use proc_macro2::Span; use proc_macro2::TokenStream; impl MetricsDecl { - #[allow(unused)] - fn agg_from_metrics_token_stream__OLD( - &self, - agg: &AggregationModItem, - inp: &MetricsModItem, - ) -> syn::Result { - let struct_name = syn::Ident::new(&agg.struct_name, Span::call_site()); - let inp_struct_name = syn::Ident::new(&inp.struct_name, Span::call_site()); - let (fields_counters_decl, fields_counters_init, ingest_counters) = { - let fields_decl = inp - .counter_names - .iter() - .map(|x| syn::Ident::new(x, Span::call_site())) - .map(|x| quote::quote! { #x: CounterU32, }); - let fields_init = inp - .counter_names - .iter() - .map(|x| syn::Ident::new(x, Span::call_site())) - .map(|x| quote::quote! { #x: CounterU32::new(), }); - let ingest_counters = inp - .counter_names - .iter() - .map(|x| syn::Ident::new(x, Span::call_site())) - .map(|x| quote::quote! { self.#x.ingest(inp.#x); }); - (fields_decl, fields_init, ingest_counters) - }; - - let (fields_compose_decl, fields_compose_init, ingest_compose) = { - let mut fields_decl = Vec::new(); - let mut fields_init = Vec::new(); - let mut ingest = Vec::new(); - for cm1 in inp.compose_mods.iter() { - log(&format!( - "agg_from_metrics_token_stream mod Compose cm {:?}", - cm1 - )); - let mut found = false; - for cm2 in agg.compose_agg_mods.iter() { - if cm2.input == cm1.name { - if found { - let e = syn::Error::new( - Span::call_site(), - format!( - "found composition method for input Compose again {:?} {:?}", - cm1, cm2 - ), - ); - return Err(e); - } - log(&format!( - "agg_from_metrics_token_stream found pair cm1 {:?} cm2 {:?}", - cm1, cm2 - )); - // TODO difference? - let _name1 = quote::format_ident!("{}", cm1.name); - let name1 = syn::Ident::new(&cm1.name, Span::call_site()); - let name2 = syn::Ident::new(&cm2.name, Span::call_site()); - let aggtor = syn::Ident::new(&cm2.aggtor, Span::call_site()); - let ts = quote::quote! { #name2: #aggtor, }; - fields_decl.push(ts); - let ts = quote::quote! { #name2: <#aggtor>::new(), }; - fields_init.push(ts); - let ts = quote::quote! { - // let _ = &inp.#name1; - self.#name2.ingest(inp.#name1); - }; - ingest.push(ts); - found = true; - } - } - if found == false { - let e = syn::Error::new( - Span::call_site(), - format!( - "did not find composition method for input Compose {:?}", - cm1 - ), - ); - return Err(e); - } - } - (fields_decl, fields_init, ingest) - }; - - let ret = quote::quote! { - #[derive(Debug, serde::Serialize)] - pub struct #struct_name { - #(#fields_counters_decl)* - #(#fields_compose_decl)* - } - - impl #struct_name { - pub fn new() -> Self { - Self { - #(#fields_counters_init)* - #(#fields_compose_init)* - } - } - - pub fn ingest(&mut self, inp: #inp_struct_name) { - #(#ingest_counters)* - #(#ingest_compose)* - } - } - }; - Ok(ret) - } - fn agg_from_counter_names( &self, agg: &AggregationModItem, @@ -268,15 +159,23 @@ impl MetricsDecl { .iter() .map(|x| syn::Ident::new(x, Span::call_site())) .map(|x| quote::quote! { #x: CounterU32, }); + let field_decl_histolog2s = metrics + .histolog2_names + .iter() + .map(|x| syn::Ident::new(x, Span::call_site())) + .map(|x| quote::quote! { #x: HistoLog2, }); let field_decl_composes = metrics.compose_mods.iter().map(|m| { let n = syn::Ident::new(&m.name, Span::call_site()); - let ct = syn::Ident::new(&m.input, Span::call_site()); + // let ct = syn::Ident::new(&m.input, Span::call_site()); + // let ct = syn::parse_str::(&m.input).unwrap(); + let ct = &m.input; quote::quote! { #n: #ct, } }); let q1 = quote::quote! { #[derive(Debug, serde::Serialize)] pub struct #struct_name { #(#field_decl_counters)* + #(#field_decl_histolog2s)* #(#field_decl_composes)* } }; @@ -286,6 +185,11 @@ impl MetricsDecl { .iter() .map(|x| syn::Ident::new(x, Span::call_site())) .map(|x| quote::quote! { #x: CounterU32::new(), }); + let field_init_histolog2s = metrics + .histolog2_names + .iter() + .map(|x| syn::Ident::new(x, Span::call_site())) + .map(|x| quote::quote! { #x: HistoLog2::new(16), }); let field_incs_counters = metrics .counter_names .iter() @@ -305,23 +209,48 @@ impl MetricsDecl { .map(|x| { quote::quote! { self.#x.ingest(inp.#x); } }); - let flatten_prom_counters = metrics - .counter_names + let fields_histlog2s_ingest = metrics + .histolog2_names .iter() - .map(|x| (syn::Ident::new(x, Span::call_site()), x)) - .map(|(x, y)| { - quote::quote! { - ret.push((stringify!(#x).into(), self.#x.to_u32() as u64)); - } + .map(|x| syn::Ident::new(x, Span::call_site())) + .map(|x| { + quote::quote! { self.#x.ingest_upstream(inp.#x); } }); + let flatten_prom_counters = metrics.counter_names.iter().map(|n| { + let id = syn::Ident::new(&n, Span::call_site()); + quote::quote! { + let n = format!("{}_{}", name, #n); + ret.push(format!("# TYPE {} counter", n)); + ret.push(format!("{} {}", n, self.#id.to_u32())); + } + }); + let flatten_prom_histolog2s = metrics.histolog2_names.iter().map(|n| { + let id = syn::Ident::new(n, Span::call_site()); + quote::quote! { + let n = format!("{}_{}", name, #n); + let v = self.#id.to_flatten_prometheus(&n); + ret.extend(v); + } + }); let field_init_composes = metrics.compose_mods.iter().map(|m| { let n = syn::Ident::new(&m.name, Span::call_site()); - let ct = syn::Ident::new(&m.input, Span::call_site()); + // let ct = syn::Ident::new(&m.input, Span::call_site()); + let ct = &m.input; quote::quote! { #n: #ct::new(), } }); + let field_histlog2s_get_mut = metrics.histolog2_names.iter().map(|x| { + let n = syn::Ident::new(x, Span::call_site()); + quote::quote! { + #[inline(always)] + pub fn #n(&mut self) -> &mut HistoLog2 { + &mut self.#n + } + } + }); let field_composes_get_mut = metrics.compose_mods.iter().map(|m| { let n = syn::Ident::new(&m.name, Span::call_site()); - let ct = syn::Ident::new(&m.input, Span::call_site()); + // let ct = syn::Ident::new(&m.input, Span::call_site()); + let ct = &m.input; quote::quote! { #[inline(always)] pub fn #n(&mut self) -> &mut #ct { @@ -331,18 +260,17 @@ impl MetricsDecl { }); let fields_composes_ingest = metrics.compose_mods.iter().map(|m| { let n = syn::Ident::new(&m.name, Span::call_site()); - let _ct = syn::Ident::new(&m.input, Span::call_site()); quote::quote! { self.#n.ingest(inp.#n); } }); let flatten_prom_composes = metrics.compose_mods.iter().map(|m| { - let n = syn::Ident::new(&m.name, Span::call_site()); + let n = &m.name; + let id = syn::Ident::new(&n, Span::call_site()); quote::quote! { - let v = self.#n.to_flatten_prometheus(); - for e in v { - ret.push((format!("{}_{}", stringify!(#n), e.0), e.1)); - }; + let n = format!("{}_{}", name, #n); + let v = self.#id.to_flatten_prometheus(&n); + ret.extend(v); } }); @@ -351,6 +279,7 @@ impl MetricsDecl { pub fn new() -> Self { Self { #(#field_init_counters)* + #(#field_init_histolog2s)* #(#field_init_composes)* } } @@ -359,16 +288,19 @@ impl MetricsDecl { } pub fn ingest(&mut self, inp: #struct_name) { #(#fields_counters_ingest)* + #(#fields_histlog2s_ingest)* #(#fields_composes_ingest)* } - pub fn to_flatten_prometheus(&self) -> Vec<(String, u64)> { + pub fn to_flatten_prometheus(&self, name: &str) -> Vec { let mut ret = Vec::new(); #(#flatten_prom_composes)* #(#flatten_prom_counters)* + #(#flatten_prom_histolog2s)* ret } #(#field_incs_counters)* + #(#field_histlog2s_get_mut)* #(#field_composes_get_mut)* } }; @@ -395,9 +327,13 @@ impl MetricsDecl { pub(super) fn to_code(&self) -> syn::Result { let mods1 = self.metrics_all_token_stream()?; let aggs = self.agg_all_token_stream()?; + let decl_js = serde_json::to_string(self).unwrap(); let ret = quote::quote! { #mods1 #aggs + pub fn decl_json() -> String { + #decl_js.into() + } }; Ok(ret) }