daqbuf/mettrics-macros
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add log2 histo

Browse Source
This commit is contained in:
Dominik Werder
2025-04-10 11:55:52 +02:00
parent 16ce869cef
commit 7edab8f24a
3 changed files with 164 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "mettrics-macros"
version = "0.0.5"
version = "0.0.6"
description = "Macros for mettrics crate."
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
license = "GPL-3.0-only"
@@ -14,3 +14,5 @@ proc-macro2 = "1"
syn = { version = "2", features = ["full", "parsing"] }
quote = "1"
prettyplease = "0.2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

123
src/make_metrics.rs
View File

@@ -4,9 +4,57 @@ mod resolve;
use crate::log::log;
use proc_macro2::Span;
use proc_macro2::TokenStream;
use std::fmt;
use syn::ext::IdentExt;
use syn::parse::ParseStream;
#[allow(unused)]
#[derive(Debug)]
pub enum Error {
Deser,
}
impl fmt::Display for Error {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(self, fmt)
}
}
#[allow(non_snake_case)]
mod serde_syn_Path {
use std::fmt;
pub fn serialize<S>(val: &syn::Path, ser: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let s = quote::quote! { #val }.to_string();
ser.serialize_str(&s)
}
struct Vis;
impl<'a> serde::de::Visitor<'a> for Vis {
type Value = syn::Path;
fn expecting(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "expect a path")
}
fn visit_str<E>(self, inp: &str) -> Result<Self::Value, E> {
let val = syn::parse_str::<syn::Path>(inp).expect("a syn parseable path");
Ok(val)
}
}
pub fn deserialize<'a, D>(de: D) -> Result<syn::Path, D::Error>
where
D: serde::Deserializer<'a>,
{
de.deserialize_str(Vis)
}
}
#[allow(unused)]
fn discard_rest(inp: ParseStream) {
inp.step(|c| {
@@ -50,7 +98,7 @@ impl syn::parse::Parse for MetricsStructNameItem {
}
}
#[derive(Debug)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct ComposeAggModItem {
input: String,
aggtor: String,
@@ -70,7 +118,8 @@ impl syn::parse::Parse for ComposeAggModItem {
if item.ident.to_string() == "Input" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
input = Some(tp.path.get_ident().unwrap().to_string());
input =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -80,7 +129,8 @@ impl syn::parse::Parse for ComposeAggModItem {
} else if item.ident.to_string() == "Aggtor" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
aggtor = Some(tp.path.get_ident().unwrap().to_string());
aggtor =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -90,7 +140,8 @@ impl syn::parse::Parse for ComposeAggModItem {
} else if item.ident.to_string() == "Name" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
name = Some(tp.path.get_ident().unwrap().to_string());
name =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -110,15 +161,15 @@ impl syn::parse::Parse for ComposeAggModItem {
}
}
let ret = Self {
input: input.unwrap(),
aggtor: aggtor.unwrap(),
name: name.unwrap(),
input: input.expect("type Input"),
aggtor: aggtor.expect("type Aggtor"),
name: name.expect("type Name"),
};
Ok(ret)
}
}
#[derive(Debug)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct AggregationModItem {
struct_name: String,
input: String,
@@ -140,7 +191,8 @@ impl syn::parse::Parse for AggregationModItem {
if item.ident.to_string() == "StructName" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
struct_name = Some(tp.path.get_ident().unwrap().to_string());
struct_name =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -150,7 +202,8 @@ impl syn::parse::Parse for AggregationModItem {
} else if item.ident.to_string() == "Input" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
input = Some(tp.path.get_ident().unwrap().to_string());
input =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -177,20 +230,32 @@ impl syn::parse::Parse for AggregationModItem {
}
}
let ret = Self {
struct_name: struct_name.unwrap(),
input: input.unwrap(),
struct_name: struct_name.expect("type StructName"),
input: input.expect("type Input"),
compose_agg_mods,
};
Ok(ret)
}
}
#[derive(Debug)]
#[derive(serde::Serialize, serde::Deserialize)]
struct ComposeModItem {
input: String,
#[serde(with = "serde_syn_Path")]
input: syn::Path,
name: String,
}
impl fmt::Debug for ComposeModItem {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
let input = &self.input;
let input = quote::quote! { #input }.to_string();
fmt.debug_struct("ComposeModItem")
.field("input", &input)
.field("name", &self.name)
.finish()
}
}
impl syn::parse::Parse for ComposeModItem {
fn parse(inp: ParseStream) -> syn::Result<Self> {
let mut input = None;
@@ -205,7 +270,7 @@ impl syn::parse::Parse for ComposeModItem {
if item.ident.to_string() == "Input" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
input = Some(tp.path.get_ident().unwrap().to_string());
input = Some(tp.path.clone());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -215,7 +280,8 @@ impl syn::parse::Parse for ComposeModItem {
} else if item.ident.to_string() == "Name" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
name = Some(tp.path.get_ident().unwrap().to_string());
name =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -235,17 +301,18 @@ impl syn::parse::Parse for ComposeModItem {
}
}
let ret = Self {
input: input.unwrap(),
name: name.unwrap(),
input: input.expect("type Input"),
name: name.expect("type Name"),
};
Ok(ret)
}
}
#[derive(Debug)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct MetricsModItem {
struct_name: String,
counter_names: Vec<String>,
histolog2_names: Vec<String>,
compose_mods: Vec<ComposeModItem>,
}
@@ -255,6 +322,7 @@ impl syn::parse::Parse for MetricsModItem {
fn parse(inp: ParseStream) -> syn::Result<Self> {
let mut struct_name = None;
let mut counter_names = Vec::new();
let mut histolog2_names = Vec::new();
let mut compose_mods = Vec::new();
log(&format!("MetricsModItem inp 1 {:?}", inp));
let item = inp.parse::<syn::ItemMod>()?;
@@ -266,7 +334,8 @@ impl syn::parse::Parse for MetricsModItem {
if item.ident.to_string() == "StructName" {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
struct_name = Some(tp.path.get_ident().unwrap().to_string());
struct_name =
Some(tp.path.get_ident().expect("path-ident").to_string());
}
_ => {
let e = inp.error(format!("expect a path type"));
@@ -302,8 +371,13 @@ impl syn::parse::Parse for MetricsModItem {
let s = var.ident.to_string();
counter_names.push(s);
}
} else if idn == "histolog2s" {
for var in vars {
let s = var.ident.to_string();
histolog2_names.push(s);
}
} else {
let e = inp.error(format!("expect enum `values` or `counters`"));
let e = inp.error(format!("expect enum `counters` or `histolog2s`"));
return Err(e);
}
}
@@ -315,15 +389,16 @@ impl syn::parse::Parse for MetricsModItem {
}
}
let ret = Self {
struct_name: struct_name.unwrap(),
struct_name: struct_name.expect("type StructName"),
counter_names,
histolog2_names,
compose_mods,
};
Ok(ret)
}
}
#[derive(Debug)]
#[derive(Debug, serde::Serialize)]
pub(crate) struct MetricsDecl {
metrics_mods: Vec<MetricsModItem>,
agg_mods: Vec<AggregationModItem>,
@@ -378,7 +453,7 @@ impl syn::parse::Parse for MetricsDecl {
match item.ty.as_ref() {
syn::Type::Path(tp) => {
let s1 = item.ident.to_string();
let s2 = tp.path.get_ident().unwrap().to_string();
let s2 = tp.path.get_ident().expect("path-ident").to_string();
if s1 == "StructName" {
log(&format!("have {:?} {:?}", s1, s2));
// struct_name = Some(s2);

188
src/make_metrics/codegen.rs
View File

@@ -1,120 +1,11 @@
use super::AggregationModItem;
use super::MetricsDecl;
use super::MetricsModItem;
use crate::log::log;
use crate::log::log_ts;
use proc_macro2::Span;
use proc_macro2::TokenStream;
impl MetricsDecl {
#[allow(unused)]
fn agg_from_metrics_token_stream__OLD(
&self,
agg: &AggregationModItem,
inp: &MetricsModItem,
) -> syn::Result<TokenStream> {
let struct_name = syn::Ident::new(&agg.struct_name, Span::call_site());
let inp_struct_name = syn::Ident::new(&inp.struct_name, Span::call_site());
let (fields_counters_decl, fields_counters_init, ingest_counters) = {
let fields_decl = inp
.counter_names
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: CounterU32, });
let fields_init = inp
.counter_names
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: CounterU32::new(), });
let ingest_counters = inp
.counter_names
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { self.#x.ingest(inp.#x); });
(fields_decl, fields_init, ingest_counters)
};
let (fields_compose_decl, fields_compose_init, ingest_compose) = {
let mut fields_decl = Vec::new();
let mut fields_init = Vec::new();
let mut ingest = Vec::new();
for cm1 in inp.compose_mods.iter() {
log(&format!(
"agg_from_metrics_token_stream mod Compose cm {:?}",
cm1
));
let mut found = false;
for cm2 in agg.compose_agg_mods.iter() {
if cm2.input == cm1.name {
if found {
let e = syn::Error::new(
Span::call_site(),
format!(
"found composition method for input Compose again {:?} {:?}",
cm1, cm2
),
);
return Err(e);
}
log(&format!(
"agg_from_metrics_token_stream found pair cm1 {:?} cm2 {:?}",
cm1, cm2
));
// TODO difference?
let _name1 = quote::format_ident!("{}", cm1.name);
let name1 = syn::Ident::new(&cm1.name, Span::call_site());
let name2 = syn::Ident::new(&cm2.name, Span::call_site());
let aggtor = syn::Ident::new(&cm2.aggtor, Span::call_site());
let ts = quote::quote! { #name2: #aggtor, };
fields_decl.push(ts);
let ts = quote::quote! { #name2: <#aggtor>::new(), };
fields_init.push(ts);
let ts = quote::quote! {
// let _ = &inp.#name1;
self.#name2.ingest(inp.#name1);
};
ingest.push(ts);
found = true;
}
}
if found == false {
let e = syn::Error::new(
Span::call_site(),
format!(
"did not find composition method for input Compose {:?}",
cm1
),
);
return Err(e);
}
}
(fields_decl, fields_init, ingest)
};
let ret = quote::quote! {
#[derive(Debug, serde::Serialize)]
pub struct #struct_name {
#(#fields_counters_decl)*
#(#fields_compose_decl)*
}
impl #struct_name {
pub fn new() -> Self {
Self {
#(#fields_counters_init)*
#(#fields_compose_init)*
}
}
pub fn ingest(&mut self, inp: #inp_struct_name) {
#(#ingest_counters)*
#(#ingest_compose)*
}
}
};
Ok(ret)
}
fn agg_from_counter_names(
&self,
agg: &AggregationModItem,
@@ -268,15 +159,23 @@ impl MetricsDecl {
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: CounterU32, });
let field_decl_histolog2s = metrics
.histolog2_names
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: HistoLog2, });
let field_decl_composes = metrics.compose_mods.iter().map(|m| {
let n = syn::Ident::new(&m.name, Span::call_site());
let ct = syn::Ident::new(&m.input, Span::call_site());
// let ct = syn::Ident::new(&m.input, Span::call_site());
// let ct = syn::parse_str::<syn::Path>(&m.input).unwrap();
let ct = &m.input;
quote::quote! { #n: #ct, }
});
let q1 = quote::quote! {
#[derive(Debug, serde::Serialize)]
pub struct #struct_name {
#(#field_decl_counters)*
#(#field_decl_histolog2s)*
#(#field_decl_composes)*
}
};
@@ -286,6 +185,11 @@ impl MetricsDecl {
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: CounterU32::new(), });
let field_init_histolog2s = metrics
.histolog2_names
.iter()
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| quote::quote! { #x: HistoLog2::new(16), });
let field_incs_counters = metrics
.counter_names
.iter()
@@ -305,23 +209,48 @@ impl MetricsDecl {
.map(|x| {
quote::quote! { self.#x.ingest(inp.#x); }
});
let flatten_prom_counters = metrics
.counter_names
let fields_histlog2s_ingest = metrics
.histolog2_names
.iter()
.map(|x| (syn::Ident::new(x, Span::call_site()), x))
.map(|(x, y)| {
quote::quote! {
ret.push((stringify!(#x).into(), self.#x.to_u32() as u64));
}
.map(|x| syn::Ident::new(x, Span::call_site()))
.map(|x| {
quote::quote! { self.#x.ingest_upstream(inp.#x); }
});
let flatten_prom_counters = metrics.counter_names.iter().map(|n| {
let id = syn::Ident::new(&n, Span::call_site());
quote::quote! {
let n = format!("{}_{}", name, #n);
ret.push(format!("# TYPE {} counter", n));
ret.push(format!("{} {}", n, self.#id.to_u32()));
}
});
let flatten_prom_histolog2s = metrics.histolog2_names.iter().map(|n| {
let id = syn::Ident::new(n, Span::call_site());
quote::quote! {
let n = format!("{}_{}", name, #n);
let v = self.#id.to_flatten_prometheus(&n);
ret.extend(v);
}
});
let field_init_composes = metrics.compose_mods.iter().map(|m| {
let n = syn::Ident::new(&m.name, Span::call_site());
let ct = syn::Ident::new(&m.input, Span::call_site());
// let ct = syn::Ident::new(&m.input, Span::call_site());
let ct = &m.input;
quote::quote! { #n: #ct::new(), }
});
let field_histlog2s_get_mut = metrics.histolog2_names.iter().map(|x| {
let n = syn::Ident::new(x, Span::call_site());
quote::quote! {
#[inline(always)]
pub fn #n(&mut self) -> &mut HistoLog2 {
&mut self.#n
}
}
});
let field_composes_get_mut = metrics.compose_mods.iter().map(|m| {
let n = syn::Ident::new(&m.name, Span::call_site());
let ct = syn::Ident::new(&m.input, Span::call_site());
// let ct = syn::Ident::new(&m.input, Span::call_site());
let ct = &m.input;
quote::quote! {
#[inline(always)]
pub fn #n(&mut self) -> &mut #ct {
@@ -331,18 +260,17 @@ impl MetricsDecl {
});
let fields_composes_ingest = metrics.compose_mods.iter().map(|m| {
let n = syn::Ident::new(&m.name, Span::call_site());
let _ct = syn::Ident::new(&m.input, Span::call_site());
quote::quote! {
self.#n.ingest(inp.#n);
}
});
let flatten_prom_composes = metrics.compose_mods.iter().map(|m| {
let n = syn::Ident::new(&m.name, Span::call_site());
let n = &m.name;
let id = syn::Ident::new(&n, Span::call_site());
quote::quote! {
let v = self.#n.to_flatten_prometheus();
for e in v {
ret.push((format!("{}_{}", stringify!(#n), e.0), e.1));
};
let n = format!("{}_{}", name, #n);
let v = self.#id.to_flatten_prometheus(&n);
ret.extend(v);
}
});
@@ -351,6 +279,7 @@ impl MetricsDecl {
pub fn new() -> Self {
Self {
#(#field_init_counters)*
#(#field_init_histolog2s)*
#(#field_init_composes)*
}
}
@@ -359,16 +288,19 @@ impl MetricsDecl {
}
pub fn ingest(&mut self, inp: #struct_name) {
#(#fields_counters_ingest)*
#(#fields_histlog2s_ingest)*
#(#fields_composes_ingest)*
}
pub fn to_flatten_prometheus(&self) -> Vec<(String, u64)> {
pub fn to_flatten_prometheus(&self, name: &str) -> Vec<String> {
let mut ret = Vec::new();
#(#flatten_prom_composes)*
#(#flatten_prom_counters)*
#(#flatten_prom_histolog2s)*
ret
}
#(#field_incs_counters)*
#(#field_histlog2s_get_mut)*
#(#field_composes_get_mut)*
}
};
@@ -395,9 +327,13 @@ impl MetricsDecl {
pub(super) fn to_code(&self) -> syn::Result<TokenStream> {
let mods1 = self.metrics_all_token_stream()?;
let aggs = self.agg_all_token_stream()?;
let decl_js = serde_json::to_string(self).unwrap();
let ret = quote::quote! {
#mods1
#aggs
pub fn decl_json() -> String {
#decl_js.into()
}
};
Ok(ret)
}