pub use rand_xoshiro; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; const US: u64 = 1000; const MS: u64 = US * 1000; const SEC: u64 = MS * 1000; pub type EMA = Ema32; #[derive(Clone, Debug)] pub struct Ema32 { ema: f32, emv: f32, k: f32, update_count: u64, } impl Ema32 { pub fn with_k(k: f32) -> Self { Self { ema: 0.0, emv: 0.0, k, update_count: 0, } } pub fn with_ema(ema: f32) -> Self { Self { ema, emv: 0.0, k: 0.05, update_count: 0, } } pub fn default() -> Self { Self { ema: 0.0, emv: 0.0, k: 0.05, update_count: 0, } } #[inline(always)] pub fn update(&mut self, v: V) where V: Into, { self.update_count += 1; let k = self.k; let dv = v.into() - self.ema; self.ema += k * dv; self.emv = (1f32 - k) * (self.emv + k * dv * dv); } pub fn update_count(&self) -> u64 { self.update_count } pub fn ema(&self) -> f32 { self.ema } pub fn emv(&self) -> f32 { self.emv } pub fn k(&self) -> f32 { self.k } } #[derive(Clone, Debug)] pub struct Ema64 { ema: f64, emv: f64, k: f64, update_count: u64, } impl Ema64 { pub fn with_k(k: f64) -> Self { Self { ema: 0.0, emv: 0.0, k, update_count: 0, } } pub fn with_ema(ema: f64, k: f64) -> Self { Self { ema, emv: 0.0, k, update_count: 0, } } pub fn default() -> Self { Self { ema: 0.0, emv: 0.0, k: 0.05, update_count: 0, } } #[inline(always)] pub fn update(&mut self, v: V) where V: Into, { self.update_count += 1; let k = self.k; let dv = v.into() - self.ema; self.ema += k * dv; self.emv = (1f64 - k) * (self.emv + k * dv * dv); } pub fn update_count(&self) -> u64 { self.update_count } pub fn ema(&self) -> f64 { self.ema } pub fn emv(&self) -> f64 { self.emv } pub fn k(&self) -> f64 { self.k } } pub struct CheckEvery { ts_last: Instant, dt: Duration, } impl CheckEvery { pub fn new(dt: Duration) -> Self { Self { ts_last: Instant::now(), dt, } } pub fn is_elapsed_now(&mut self) -> f32 { let now = Instant::now(); let dt = now.duration_since(self.ts_last); if dt >= self.dt { self.ts_last = now; dt.as_secs_f32() } else { -16f32 } } } #[derive(Clone, Debug)] pub struct IntervalEma { tslast: Option, ema: EMA, } impl IntervalEma { pub fn new() -> Self { Self { tslast: None, ema: EMA::default(), } } pub fn tick(&mut self, tsnow: Instant) { match self.tslast { Some(tslast) => { let dt = tsnow.duration_since(tslast); self.tslast = Some(tsnow); self.ema.update(dt.as_secs_f32()); } None => { self.tslast = Some(tsnow); } } } pub fn ema_preview(&self, tsnow: Instant) -> Option { match self.tslast { Some(tslast) => { let dt = tsnow.duration_since(tslast); let v = dt.as_secs_f32(); let dv = v - self.ema.ema; Some(self.ema.ema + self.ema.k * dv) } None => None, } } pub fn ema(&self) -> &EMA { &self.ema } } pub struct XorShift32 { state: u32, } impl XorShift32 { pub fn new(state: u32) -> Self { Self { state } } pub fn new_from_time() -> Self { use std::time::SystemTime; Self::new( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() .subsec_micros(), ) } pub fn next(&mut self) -> u32 { let mut x = self.state; x ^= x << 13; x ^= x >> 17; x ^= x << 5; self.state = x; x } } stats_proc::stats_struct!(( stats_struct( name(CaProtoStats), prefix(ca_proto), counters( tcp_recv_count, tcp_recv_bytes, protocol_issue, payload_std_too_large, payload_ext_but_small, payload_ext_very_large, ), histolog2s(payload_size, data_count,), ), stats_struct( name(CaConnSetStats), prefix(connset), counters( channel_add, channel_status_series_found, channel_health_timeout, ioc_search_start, ioc_addr_found, ioc_addr_not_found, ioc_addr_result_for_unknown_channel, ca_conn_task_begin, ca_conn_task_done, ca_conn_task_join_done_ok, ca_conn_task_join_done_err, ca_conn_task_join_err, ca_conn_eos_ok, ca_conn_eos_unexpected, response_tx_fail, try_push_ca_conn_cmds_sent, try_push_ca_conn_cmds_full, try_push_ca_conn_cmds_closed, logic_error, logic_issue, ready_for_end_of_stream, ready_for_end_of_stream_with_progress, poll_fn_begin, poll_loop_begin, poll_pending, poll_reloop, poll_no_progress_no_pending, handle_add_channel_with_addr, create_ca_conn, command_reply_fail, ), values( storage_insert_queue_len, storage_insert_tx_len, channel_info_query_queue_len, channel_info_query_sender_len, channel_info_res_tx_len, find_ioc_query_sender_len, ca_conn_res_tx_len, channel_unknown_address, channel_search_pending, channel_no_address, channel_unassigned, channel_assigned, channel_connected, channel_maybe_wrong_address, channel_assigned_without_health_update, channel_rogue, ), ), // agg(name(CaConnSetStatsAgg), parent(CaConnSetStats)), // diff(name(CaConnSetStatsDiff), input(CaConnSetStats)), stats_struct( name(SeriesByChannelStats), prefix(seriesbychannel), counters(res_tx_fail, res_tx_timeout, recv_batch, recv_items,), ), stats_struct( name(InsertWorkerStats), prefix(insert_worker), counters( item_recv, inserted_values, inserted_connection_status, inserted_channel_status, fraction_drop, inserted_mute, inserted_interval, inserted_channel_info, inserted_binned, db_overload, db_timeout, db_unavailable, db_error, query_error, inserts_msp, inserts_msp_grid, inserts_value, ratelimit_drop, worker_start, worker_finish, ), histolog2s(item_lat_net_worker, item_lat_net_store,), ), stats_struct( name(IocFinderStats), prefix(ioc_finder), counters( dbsearcher_batch_recv, dbsearcher_item_recv, dbsearcher_select_res_0, dbsearcher_select_error_len_mismatch, dbsearcher_batch_send, dbsearcher_item_send, ca_udp_error, ca_udp_warn, ca_udp_unaccounted_data, ca_udp_batch_created, ca_udp_io_error, ca_udp_io_empty, ca_udp_io_recv, ca_udp_first_msg_not_version, ca_udp_recv_result, ca_udp_recv_timeout, ca_udp_logic_error, ), values(db_lookup_workers,) ), )); stats_proc::stats_struct!(( stats_struct( name(CaConnStats), prefix(caconn), counters( insert_item_create, inserts_val, inserts_msp, inserts_msp_grid, inserts_queue_pop_for_global, inserts_queue_push, inserts_queue_drop, insert_item_queue_pressure, insert_item_queue_full, channel_fast_item_drop, logic_error, // TODO maybe rename: this is now only the recv of the intermediate queue: store_worker_item_recv, // TODO rename to make clear that this drop is voluntary because of user config choice: // store_worker_fraction_drop, // store_worker_ratelimit_drop, // store_worker_insert_done, // store_worker_insert_binned_done, // store_worker_insert_overload, // store_worker_insert_timeout, // store_worker_insert_unavailable, // store_worker_insert_error, connection_status_insert_done, channel_status_insert_done, channel_info_insert_done, ivl_insert_done, mute_insert_done, poll_count, loop1_count, loop2_count, loop3_count, loop4_count, command_can_not_reply, time_handle_conn_listen, time_handle_peer_ready, time_check_channels_state_init, time_handle_event_add_res, tcp_connected, get_series_id_ok, item_count, stream_ready, stream_pending, channel_all_count, channel_alive_count, channel_not_alive_count, channel_series_lookup_already_pending, ping_start, ping_no_proto, pong_timeout, poll_fn_begin, poll_loop_begin, poll_reloop, poll_pending, poll_no_progress_no_pending, poll_reloops_8, poll_reloops_64, poll_reloops_512, poll_wake_break, storage_queue_send, storage_queue_pending, storage_queue_above_8, storage_queue_above_32, storage_queue_above_128, event_add_res_recv, ), values(inter_ivl_ema), histolog2s(pong_recv_lat, ca_ts_off,), ), agg(name(CaConnStatsAgg), parent(CaConnStats)), diff(name(CaConnStatsAggDiff), input(CaConnStatsAgg)), )); stats_proc::stats_struct!(( stats_struct( name(DaemonStats), prefix(daemon), counters( critical_error, todo_mark, ticker_token_acquire_error, ticker_token_release_error, handle_timer_tick_count, ioc_search_err, ioc_search_some, ioc_search_none, lookupaddr_ok, events, event_ca_conn, ca_conn_status_done, ca_conn_status_feedback_timeout, ca_conn_status_feedback_recv, ca_conn_status_feedback_no_dst, ca_echo_timeout_total, caconn_done_channel_state_reset, insert_worker_spawned, insert_worker_join_ok, insert_worker_join_ok_err, insert_worker_join_err, caconnset_health_response, ), values( channel_unknown_address, channel_search_pending, channel_with_address, channel_no_address, connset_health_lat_ema, ), ), agg(name(DaemonStatsAgg), parent(DaemonStats)), diff(name(DaemonStatsAggDiff), input(DaemonStatsAgg)), )); stats_proc::stats_struct!(( stats_struct(name(TestStats0), counters(count0,), values(val0),), diff(name(TestStats0Diff), input(TestStats0)), agg(name(TestStats0Agg), parent(TestStats0)), diff(name(TestStats0AggDiff), input(TestStats0Agg)), )); #[test] fn test0_diff() { let stats_a = TestStats0::new(); stats_a.count0().inc(); stats_a.val0().set(43); let stats_b = stats_a.snapshot(); stats_b.count0().inc(); stats_b.count0().inc(); stats_b.count0().inc(); let diff = TestStats0Diff::diff_from(&stats_a, &stats_b); assert_eq!(diff.count0.load(), 3); } pub fn xoshiro_from_time() -> rand_xoshiro::Xoshiro128PlusPlus { use rand_xoshiro::rand_core::SeedableRng; use std::time::SystemTime; let a = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() .subsec_nanos() as u64; let b = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() .subsec_nanos() as u64; rand_xoshiro::Xoshiro128PlusPlus::seed_from_u64(a << 32 ^ b) }