daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Cleaning and transition more stats counters

Browse Source
This commit is contained in:
Dominik Werder
2025-04-28 17:12:46 +02:00
parent a653816ed2
commit 265f4b9bd9
25 changed files with 265 additions and 2116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
netfetch/src/ca.rs
View File

@@ -8,7 +8,6 @@ pub mod findioc;
pub mod search;
pub mod statemap;
use crate::metrics::ExtraInsertsConf;
use futures_util::Future;
use futures_util::FutureExt;
use log::*;

4
netfetch/src/ca/beacons.rs
View File

@@ -41,7 +41,6 @@ pub async fn listen_beacons(
worker_tx: Sender<ChannelInfoQuery>,
backend: String,
) -> Result<(), Error> {
let stnow = SystemTime::now();
let channel = "epics-ca-beacons".to_string();
let scalar_type = ScalarType::U64;
let shape = Shape::Scalar;
@@ -56,6 +55,7 @@ pub async fn listen_beacons(
};
worker_tx.send(qu).await?;
let chinfo = rx.recv().await??;
let _ = chinfo;
// TODO
// let mut writer = SeriesWriter::new(chinfo.series.to_series());
// let mut deque = VecDeque::new();
@@ -91,6 +91,8 @@ pub async fn listen_beacons(
let ts_local = ts;
let blob = addr_u32 as i64;
let val = DataValue::Scalar(ScalarValue::I64(blob));
let _ = ts_local;
let _ = val;
// writer.write(ts, ts_local, val, &mut deque)?;
}
}

113
netfetch/src/ca/conn.rs
View File

@@ -781,6 +781,7 @@ enum CaConnState {
PeerReady,
Shutdown(EndOfStreamReason),
EndOfStream,
MetricsEmitted,
}
impl fmt::Debug for CaConnState {
@@ -793,6 +794,7 @@ impl fmt::Debug for CaConnState {
Self::PeerReady => fmt.debug_tuple("PeerReady").finish(),
Self::Shutdown(v0) => fmt.debug_tuple("Shutdown").field(v0).finish(),
Self::EndOfStream => fmt.debug_tuple("EndOfStream").finish(),
Self::MetricsEmitted => fmt.debug_tuple("MetricsEmitted").finish(),
}
}
}
@@ -1101,7 +1103,6 @@ pub struct CaConn {
ca_conn_event_out_queue: VecDeque<CaConnEvent>,
ca_conn_event_out_queue_max: usize,
thr_msg_poll: ThrottleTrace,
ca_proto_stats: Arc<CaProtoStats>,
rng: Xoshiro128PlusPlus,
channel_info_query_qu: VecDeque<ChannelInfoQuery>,
channel_info_query_tx: Pin<Box<SenderPolling<ChannelInfoQuery>>>,
@@ -1118,6 +1119,7 @@ pub struct CaConn {
ts_channel_status_pong_last: Instant,
mett: stats::mett::CaConnMetrics,
metrics_emit_last: Instant,
fionread_last: u32,
}
impl Drop for CaConn {
@@ -1135,7 +1137,6 @@ impl CaConn {
iqtx: InsertQueuesTx,
channel_info_query_tx: Sender<ChannelInfoQuery>,
stats: Arc<CaConnStats>,
ca_proto_stats: Arc<CaProtoStats>,
) -> Self {
let tsnow = Instant::now();
let (cq_tx, cq_rx) = async_channel::bounded(32);
@@ -1174,7 +1175,6 @@ impl CaConn {
ca_conn_event_out_queue: VecDeque::new(),
ca_conn_event_out_queue_max: 2000,
thr_msg_poll: ThrottleTrace::new(Duration::from_millis(2000)),
ca_proto_stats,
rng,
channel_info_query_qu: VecDeque::new(),
channel_info_query_tx: Box::pin(SenderPolling::new(channel_info_query_tx)),
@@ -1188,6 +1188,7 @@ impl CaConn {
ts_channel_status_pong_last: tsnow,
mett: stats::mett::CaConnMetrics::new(),
metrics_emit_last: tsnow,
fionread_last: 0,
}
}
@@ -1528,12 +1529,14 @@ impl CaConn {
if dbg_chn_cid {
info!("send out EventAdd for {cid:?}");
}
let ty = CaMsgTy::EventAdd(EventAdd {
sid: st2.channel.sid.to_u32(),
data_type: st2.channel.ca_dbr_type,
data_count: st2.channel.ca_dbr_count,
subid: subid.to_u32(),
});
let data_count = st2.channel.ca_dbr_count;
let _data_count = 0;
let ty = CaMsgTy::EventAdd(EventAdd::new(
st2.channel.ca_dbr_type,
data_count,
st2.channel.sid.to_u32(),
subid.to_u32(),
));
let msg = CaMsg::from_ty_ts(ty, self.poll_tsnow);
let proto = self.proto.as_mut().unwrap();
proto.push_out(msg);
@@ -1802,7 +1805,12 @@ impl CaConn {
Ok(())
}
fn handle_event_add_res(&mut self, ev: proto::EventAddRes, tsnow: Instant) -> Result<(), Error> {
fn handle_event_add_res(
&mut self,
ev: proto::EventAddRes,
tsnow: Instant,
tscaproto: Instant,
) -> Result<(), Error> {
let subid = Subid(ev.subid);
// TODO handle subid-not-found which can also be peer error:
let cid = if let Some(x) = self.cid_by_subid.get(&subid) {
@@ -1910,6 +1918,7 @@ impl CaConn {
iqdqs,
tsnow,
stnow,
tscaproto,
ch_conf.use_ioc_time(),
stats,
&mut self.rng,
@@ -1942,6 +1951,7 @@ impl CaConn {
iqdqs,
tsnow,
stnow,
tscaproto,
ch_conf.use_ioc_time(),
stats,
&mut self.rng,
@@ -2049,8 +2059,8 @@ impl CaConn {
fn handle_read_notify_res(
&mut self,
ev: proto::ReadNotifyRes,
camsg_ts: Instant,
tsnow: Instant,
tscaproto: Instant,
) -> Result<(), Error> {
// trace!("handle_read_notify_res {ev:?}");
// TODO can not rely on the SID in the response.
@@ -2058,10 +2068,7 @@ impl CaConn {
let ioid = Ioid(ev.ioid);
if let Some(pp) = self.handler_by_ioid.get_mut(&ioid) {
if let Some(mut fut) = pp.take() {
let camsg = CaMsg {
ty: CaMsgTy::ReadNotifyRes(ev),
ts: camsg_ts,
};
let camsg = CaMsg::from_ty_ts(CaMsgTy::ReadNotifyRes(ev), tscaproto);
fut.as_mut().camsg(camsg, self)?;
Ok(())
} else {
@@ -2119,6 +2126,7 @@ impl CaConn {
iqdqs,
stnow,
tsnow,
tscaproto,
ch_conf.use_ioc_time(),
stats,
&mut self.rng,
@@ -2211,6 +2219,7 @@ impl CaConn {
iqdqs,
stnow,
tsnow,
tscaproto,
ch_conf.use_ioc_time(),
stats,
&mut self.rng,
@@ -2243,6 +2252,7 @@ impl CaConn {
iqdqs: &mut InsertDeques,
stnow: SystemTime,
tsnow: Instant,
tscaproto: Instant,
use_ioc_time: bool,
stats: &CaConnStats,
rng: &mut Xoshiro128PlusPlus,
@@ -2260,6 +2270,7 @@ impl CaConn {
iqdqs,
tsnow,
stnow,
tscaproto,
use_ioc_time,
stats,
rng,
@@ -2277,6 +2288,7 @@ impl CaConn {
iqdqs: &mut InsertDeques,
tsnow: Instant,
stnow: SystemTime,
tscaproto: Instant,
use_ioc_time: bool,
stats: &CaConnStats,
rng: &mut Xoshiro128PlusPlus,
@@ -2340,7 +2352,7 @@ impl CaConn {
crst.insert_item_ivl_ema.tick(tsnow);
// binwriter.ingest(tsev, value.f32_for_binning(), iqdqs)?;
{
let wres = writer.write(CaWriterValue::new(value, crst), tsnow, tsev, iqdqs)?;
let wres = writer.write(CaWriterValue::new(value, crst), tscaproto, tsev, iqdqs)?;
crst.status_emit_count += wres.nstatus() as u64;
if wres.st.accept {
crst.dw_st_last = stnow;
@@ -2595,13 +2607,11 @@ impl CaConn {
// Do not go directly into error state: need to at least attempt to close the channel and wait/timeout for reply.
let proto = self.proto.as_mut().ok_or(Error::NoProtocol)?;
let item = CaMsg {
ty: CaMsgTy::ChannelClose(ChannelClose {
sid: st2.channel.sid.0,
cid: st2.channel.cid.0,
}),
ts: tsnow,
};
let ty = CaMsgTy::ChannelClose(ChannelClose {
sid: st2.channel.sid.0,
cid: st2.channel.cid.0,
});
let item = CaMsg::from_ty_ts(ty, tsnow);
proto.push_out(item);
*chst = ChannelState::Closing(ClosingState {
tsbeg: tsnow,
@@ -2775,7 +2785,8 @@ impl CaConn {
Ready(Some(Ok(k))) => {
match k {
CaItem::Msg(camsg) => {
match &camsg.ty {
let (msgcom, ty) = camsg.into_parts();
match &ty {
CaMsgTy::Version => {
if !self.version_seen {
self.version_seen = true;
@@ -2804,7 +2815,7 @@ impl CaConn {
}
}
}
match camsg.ty {
match ty {
CaMsgTy::SearchRes(k) => {
let a = k.addr.to_be_bytes();
let addr = format!("{}.{}.{}.{}:{}", a[0], a[1], a[2], a[3], k.tcp_port);
@@ -2817,15 +2828,15 @@ impl CaConn {
cx.waker().wake_by_ref();
}
CaMsgTy::EventAddRes(ev) => {
trace4!("got EventAddRes {:?} cnt {}", camsg.ts, ev.data_count);
trace4!("got EventAddRes {:?} cnt {}", msgcom.ts(), ev.data_count);
self.mett.event_add_res_recv().inc();
Self::handle_event_add_res(self, ev, tsnow)?
Self::handle_event_add_res(self, ev, tsnow, msgcom.ts())?
}
CaMsgTy::EventAddResEmpty(ev) => {
trace4!("got EventAddResEmpty {:?}", camsg.ts);
trace4!("got EventAddResEmpty {:?}", msgcom.ts());
Self::handle_event_add_res_empty(self, ev, tsnow)?
}
CaMsgTy::ReadNotifyRes(ev) => Self::handle_read_notify_res(self, ev, camsg.ts, tsnow)?,
CaMsgTy::ReadNotifyRes(ev) => Self::handle_read_notify_res(self, ev, tsnow, msgcom.ts())?,
CaMsgTy::Echo => {
if let Some(started) = self.ioc_ping_start {
let dt = started.elapsed();
@@ -2874,11 +2885,11 @@ impl CaConn {
}
self.version_seen = true;
}
CaMsgTy::ChannelCloseRes(x) => {
self.handle_channel_close_res(x, tsnow)?;
CaMsgTy::ChannelCloseRes(ty) => {
self.handle_channel_close_res(ty, tsnow)?;
}
_ => {
warn!("Received unexpected protocol message {:?}", camsg);
warn!("Received unexpected protocol message {:?} {:?}", msgcom, ty);
}
}
}
@@ -3014,6 +3025,7 @@ impl CaConn {
}
fn handle_channel_close_res(&mut self, k: proto::ChannelCloseRes, tsnow: Instant) -> Result<(), Error> {
let _ = tsnow;
debug!("{:?}", k);
Ok(())
}
@@ -3037,6 +3049,11 @@ impl CaConn {
Ready(connect_result) => {
match connect_result {
Ok(Ok(tcp)) => {
let raw_fd = {
use std::os::fd::AsRawFd;
let raw_fd = tcp.as_raw_fd();
raw_fd
};
self.mett.tcp_connected().inc();
let addr = addr.clone();
self.emit_connection_status_item(ConnectionStatusItem {
@@ -3047,6 +3064,7 @@ impl CaConn {
self.backoff_reset();
let proto = CaProto::new(
TcpAsyncWriteRead::from(tcp),
Some(raw_fd),
self.remote_addr_dbg.to_string(),
self.opts.array_truncate,
);
@@ -3128,6 +3146,7 @@ impl CaConn {
}
CaConnState::Shutdown(..) => Ok(Ready(None)),
CaConnState::EndOfStream => Ok(Ready(None)),
CaConnState::MetricsEmitted => Ok(Ready(None)),
};
}
}
@@ -3230,6 +3249,7 @@ impl CaConn {
CaConnState::PeerReady => {}
CaConnState::Shutdown(..) => {}
CaConnState::EndOfStream => {}
CaConnState::MetricsEmitted => {}
}
self.iqdqs.housekeeping();
if self.metrics_emit_last + METRICS_EMIT_IVL <= tsnow {
@@ -3243,8 +3263,29 @@ impl CaConn {
fn metrics_emit(&mut self) {
if let Some(x) = self.proto.as_mut() {
let fionread = if let Some(rawfd) = x.get_raw_socket_fd() {
let mut v = 0;
if unsafe { libc::ioctl(rawfd, libc::FIONREAD, &mut v) } == 0 {
Some(v as u32)
} else {
None
}
} else {
None
};
let mett = x.mett();
mett.metrics_emit().inc();
if let Some(fionread) = fionread {
if fionread > self.fionread_last {
let diff = fionread - self.fionread_last;
self.fionread_last = fionread;
mett.fionread_inc().add(diff);
} else if fionread < self.fionread_last {
let diff = self.fionread_last - fionread;
self.fionread_last = fionread;
mett.fionread_dec().add(diff);
}
}
let m = mett.take_and_reset();
self.mett.proto().ingest(m);
}
@@ -3552,8 +3593,14 @@ impl Stream for CaConn {
let mut have_pending = false;
let mut have_progress = false;
if let CaConnState::EndOfStream = self.state {
if let CaConnState::MetricsEmitted = self.state {
break Ready(None);
} else if let CaConnState::EndOfStream = self.state {
self.mett.metrics_emit_final().inc();
let mett = self.mett.take_and_reset();
self.state = CaConnState::MetricsEmitted;
break Ready(Some(CaConnEvent::new_now(CaConnEventValue::Metrics(mett))));
// break Ready(None);
} else if let Some(item) = self.ca_conn_event_out_queue.pop_front() {
break Ready(Some(item));
}

111
netfetch/src/ca/connset.rs
View File

@@ -2,7 +2,6 @@ use super::conn::EndOfStreamReason;
use super::findioc::FindIocRes;
use crate::ca::conn;
use crate::ca::statemap;
use crate::ca::statemap::CaConnState;
use crate::ca::statemap::MaybeWrongAddressState;
use crate::ca::statemap::WithAddressState;
use crate::conf::CaIngestOpts;
@@ -38,7 +37,6 @@ use scywr::senderpolling::SenderPolling;
use serde::Serialize;
use series::ChannelStatusSeriesId;
use statemap::ActiveChannelState;
use statemap::CaConnStateValue;
use statemap::ChannelState;
use statemap::ChannelStateMap;
use statemap::ChannelStateValue;
@@ -50,8 +48,6 @@ use stats::CaConnSetStats;
use stats::CaConnStats;
use stats::CaProtoStats;
use stats::IocFinderStats;
use stats::rand_xoshiro::Xoshiro128PlusPlus;
use stats::rand_xoshiro::rand_core::RngCore;
use std::collections::BTreeMap;
use std::collections::VecDeque;
use std::fmt;
@@ -76,18 +72,16 @@ use tracing::Instrument;
const CHECK_CHANS_PER_TICK: usize = 10000000;
pub const SEARCH_BATCH_MAX: usize = 64;
pub const CURRENT_SEARCH_PENDING_MAX: usize = SEARCH_BATCH_MAX * 4;
const UNKNOWN_ADDRESS_STAY: Duration = Duration::from_millis(15000);
const NO_ADDRESS_STAY: Duration = Duration::from_millis(20000);
const MAYBE_WRONG_ADDRESS_STAY: Duration = Duration::from_millis(4000);
const SEARCH_PENDING_TIMEOUT: Duration = Duration::from_millis(30000);
const CHANNEL_HEALTH_TIMEOUT: Duration = Duration::from_millis(30000);
const CHANNEL_UNASSIGNED_TIMEOUT: Duration = Duration::from_millis(0);
const UNASSIGN_FOR_CONFIG_CHANGE_TIMEOUT: Duration = Duration::from_millis(1000 * 10);
const CHANNEL_MAX_WITHOUT_HEALTH_UPDATE: usize = 3000000;
macro_rules! trace2 { ($($arg:tt)*) => { if false { trace!($($arg)*); } }; }
macro_rules! trace2 { ($($arg:expr),*) => ( if false { trace!($($arg),*); } ); }
macro_rules! trace3 { ($($arg:tt)*) => { if false { trace!($($arg)*); } }; }
macro_rules! trace3 { ($($arg:expr),*) => ( if false { trace!($($arg),*); } ); }
macro_rules! trace4 { ($($arg:tt)*) => { if false { trace!($($arg)*); } }; }
@@ -139,7 +133,6 @@ impl From<Error> for ::err::Error {
pub struct CmdId(SocketAddrV4, usize);
pub struct CaConnRes {
state: CaConnState,
sender: Pin<Box<SenderPolling<ConnCommand>>>,
stats: Arc<CaConnStats>,
cmd_queue: VecDeque<ConnCommand>,
@@ -265,11 +258,8 @@ pub struct CaConnSetCtrl {
rx: Receiver<CaConnSetItem>,
stats: Arc<CaConnSetStats>,
ca_conn_stats: Arc<CaConnStats>,
ca_proto_stats: Arc<CaProtoStats>,
ioc_finder_stats: Arc<IocFinderStats>,
jh: JoinHandle<Result<(), Error>>,
rng: Xoshiro128PlusPlus,
idcnt: u32,
}
impl CaConnSetCtrl {
@@ -335,19 +325,9 @@ impl CaConnSetCtrl {
&self.ca_conn_stats
}
pub fn ca_proto_stats(&self) -> &Arc<CaProtoStats> {
&self.ca_proto_stats
}
pub fn ioc_finder_stats(&self) -> &Arc<IocFinderStats> {
&self.ioc_finder_stats
}
fn make_id(&mut self) -> u32 {
let id = self.idcnt;
self.idcnt += 1;
self.rng.next_u32() & 0xffff | (id << 16)
}
}
#[derive(Debug)]
@@ -452,11 +432,8 @@ pub struct CaConnSet {
ca_conn_stats: Arc<CaConnStats>,
ioc_finder_jh: JoinHandle<Result<(), crate::ca::finder::Error>>,
await_ca_conn_jhs: VecDeque<(SocketAddr, JoinHandle<Result<(), Error>>)>,
thr_msg_poll_1: ThrottleTrace,
thr_msg_storage_len: ThrottleTrace,
ca_proto_stats: Arc<CaProtoStats>,
rogue_channel_count: u64,
connect_fail_count: usize,
cssid_latency_max: Duration,
ca_connset_metrics: stats::mett::CaConnSetMetrics,
}
@@ -526,11 +503,8 @@ impl CaConnSet {
// connset_out_sender: SenderPolling::new(connset_out_tx),
ioc_finder_jh,
await_ca_conn_jhs: VecDeque::new(),
thr_msg_poll_1: ThrottleTrace::new(Duration::from_millis(2000)),
thr_msg_storage_len: ThrottleTrace::new(Duration::from_millis(1000)),
ca_proto_stats: ca_proto_stats.clone(),
rogue_channel_count: 0,
connect_fail_count: 0,
cssid_latency_max: Duration::from_millis(2000),
ca_connset_metrics: stats::mett::CaConnSetMetrics::new(),
};
@@ -541,11 +515,8 @@ impl CaConnSet {
rx: connset_out_rx,
stats,
ca_conn_stats,
ca_proto_stats,
ioc_finder_stats,
jh,
idcnt: 0,
rng: stats::xoshiro_from_time(),
}
}
@@ -1249,7 +1220,6 @@ impl CaConnSet {
fn handle_ca_conn_channel_removed(&mut self, addr: SocketAddr, name: String) -> Result<(), Error> {
debug!("handle_ca_conn_channel_removed {addr} {name}");
let stnow = SystemTime::now();
let name = ChannelName::new(name);
if let Some(st1) = self.channel_states.get_mut(&name) {
match &mut st1.value {
@@ -1368,7 +1338,6 @@ impl CaConnSet {
.clone()
.ok_or_else(|| Error::MissingChannelInfoChannelTx)?,
self.ca_conn_stats.clone(),
self.ca_proto_stats.clone(),
);
let conn_tx = conn.conn_command_tx();
let conn_stats = conn.stats();
@@ -1387,7 +1356,6 @@ impl CaConnSet {
let fut = fut.instrument(logspan);
let jh = tokio::spawn(fut);
let ca_conn_res = CaConnRes {
state: CaConnState::new(CaConnStateValue::Fresh),
sender: Box::pin(conn_tx.into()),
stats: conn_stats,
cmd_queue: VecDeque::new(),
@@ -1470,80 +1438,6 @@ impl CaConnSet {
}
}
async fn wait_stopped(&self) -> Result<(), Error> {
warn!("Lock for wait_stopped");
// let mut g = self.ca_conn_ress.lock().await;
// let mm = std::mem::replace(&mut *g, BTreeMap::new());
let mm: BTreeMap<SocketAddrV4, JoinHandle<Result<(), Error>>> = BTreeMap::new();
let mut jhs: VecDeque<_> = VecDeque::new();
for t in mm {
jhs.push_back(t.1.fuse());
}
loop {
let mut jh = if let Some(x) = jhs.pop_front() {
x
} else {
break;
};
futures_util::select! {
a = jh => match a {
Ok(k) => match k {
Ok(_) => {}
Err(e) => {
error!("{e:?}");
}
},
Err(e) => {
error!("{e:?}");
}
},
_b = crate::rt::sleep(Duration::from_millis(1000)).fuse() => {
jhs.push_back(jh);
info!("waiting for {} connections", jhs.len());
}
};
}
Ok(())
}
fn check_connection_states(&mut self) -> Result<(), Error> {
let tsnow = Instant::now();
for (addr, val) in &mut self.ca_conn_ress {
let state = &mut val.state;
let v = &mut state.value;
match v {
CaConnStateValue::Fresh => {
// TODO check for delta t since last issued status command.
if tsnow.duration_since(state.last_feedback) > Duration::from_millis(20000) {
error!("TODO Fresh timeout send connection-close for {addr}");
// TODO collect in metrics
// self.stats.ca_conn_status_feedback_timeout.inc();
// TODO send shutdown to this CaConn, check that we've received
// a 'shutdown' state from it. (see below)
*v = CaConnStateValue::Shutdown { since: tsnow };
}
}
CaConnStateValue::HadFeedback => {
// TODO check for delta t since last issued status command.
if tsnow.duration_since(state.last_feedback) > Duration::from_millis(20000) {
error!("TODO HadFeedback timeout send connection-close for {addr}");
// TODO collect in metrics
// self.stats.ca_conn_status_feedback_timeout.inc();
*v = CaConnStateValue::Shutdown { since: tsnow };
}
}
CaConnStateValue::Shutdown { since } => {
if tsnow.saturating_duration_since(*since) > Duration::from_millis(10000) {
// TODO collect in metrics as severe error, this would be a bug.
// self.stats.critical_error.inc();
error!("Shutdown of CaConn failed for {addr}");
}
}
}
}
Ok(())
}
fn check_channel_states(&mut self, tsnow: Instant, stnow: SystemTime) -> Result<(), Error> {
let (mut search_pending_count, mut assigned_without_health_update) = self.update_channel_state_counts();
let mut cmd_remove_channel = Vec::new();
@@ -1555,6 +1449,7 @@ impl CaConnSet {
} else {
self.channel_states.range_mut(..)
};
#[allow(unused)]
let mut st_qu_2 = VecDeque::new();
let mut lt_qu_2 = VecDeque::new();
for (i, (ch, st)) in it.enumerate() {

8
netfetch/src/ca/connset_input_merge.rs
View File

@@ -41,6 +41,10 @@ impl InputMerge {
}
}
fn todoval<T>() -> T {
todo!()
}
impl Stream for InputMerge {
type Item = CaConnSetEvent;
@@ -50,7 +54,7 @@ impl Stream for InputMerge {
let mut selfp = self.as_mut().project();
if let Some(inp) = selfp.inp3.as_mut().as_pin_mut() {
match inp.poll_next(cx) {
Ready(Some(x)) => Some(CaConnSetEvent::ConnSetCmd(todo!())),
Ready(Some(_)) => Some(CaConnSetEvent::ConnSetCmd(todoval())),
Ready(None) => {
unsafe {
// TODO what guarantees that I can drop the content here like this?
@@ -70,7 +74,7 @@ impl Stream for InputMerge {
let mut selfp = self.as_mut().project();
if let Some(inp) = selfp.inp2.as_mut().as_pin_mut() {
match inp.poll_next(cx) {
Ready(Some(x)) => Some(CaConnSetEvent::ConnSetCmd(todo!())),
Ready(Some(_)) => Some(CaConnSetEvent::ConnSetCmd(todoval())),
Ready(None) => {
unsafe {
// TODO what guarantees that I can drop the content here like this?

2
netfetch/src/ca/finder.rs
View File

@@ -40,7 +40,7 @@ fn transform_pgres(rows: Vec<PgRow>) -> VecDeque<FindIocRes> {
let n: Result<i32, _> = row.try_get(0);
let ch: Result<String, _> = row.try_get(1);
match (n, ch) {
(Ok(n), Ok(ch)) => {
(Ok(_n), Ok(ch)) => {
if let Some(addr) = row.get::<_, Option<String>>(3) {
let addr = addr.parse().map_or(None, |x| Some(x));
let item = FindIocRes {

1560
netfetch/src/ca/proto.rs
View File

File diff suppressed because it is too large Load Diff

17
netfetch/src/ca/statemap.rs
View File

@@ -24,21 +24,6 @@ pub enum CaConnStateValue {
Shutdown { since: Instant },
}
#[derive(Debug)]
pub struct CaConnState {
pub last_feedback: Instant,
pub value: CaConnStateValue,
}
impl CaConnState {
pub fn new(value: CaConnStateValue) -> Self {
Self {
last_feedback: Instant::now(),
value,
}
}
}
#[derive(Debug, Clone, Serialize)]
pub enum ConnectionStateValue {
Unknown,
@@ -243,6 +228,8 @@ impl ChannelStateMap {
}
pub fn insert(&mut self, k: ChannelName, v: ChannelState) -> Option<ChannelState> {
let _ = &self.map2;
let _ = &self.map3;
self.map.insert(k, v)
}

4
netfetch/src/conf.rs
View File

@@ -23,8 +23,6 @@ pub struct CaIngestOpts {
search: Vec<String>,
#[serde(default)]
search_blacklist: Vec<String>,
whitelist: Option<String>,
blacklist: Option<String>,
#[allow(unused)]
#[serde(default, with = "humantime_serde")]
timeout: Option<Duration>,
@@ -424,6 +422,7 @@ mod serde_replication_bool {
use serde::de;
use std::fmt;
#[allow(unused)]
pub fn serialize<S>(v: &bool, ser: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
@@ -488,6 +487,7 @@ mod serde_option_channel_read_config {
use std::fmt;
use std::time::Duration;
#[allow(unused)]
pub fn serialize<S>(v: &Option<ChannelReadConfig>, ser: S) -> Result<S::Ok, S::Error>
where
S: Serializer,

6
netfetch/src/metrics.rs
View File

@@ -135,7 +135,6 @@ pub struct StatsSet {
daemon: Arc<DaemonStats>,
ca_conn_set: Arc<CaConnSetStats>,
ca_conn: Arc<CaConnStats>,
ca_proto: Arc<CaProtoStats>,
insert_worker_stats: Arc<InsertWorkerStats>,
series_by_channel_stats: Arc<SeriesByChannelStats>,
ioc_finder_stats: Arc<IocFinderStats>,
@@ -147,7 +146,6 @@ impl StatsSet {
daemon: Arc<DaemonStats>,
ca_conn_set: Arc<CaConnSetStats>,
ca_conn: Arc<CaConnStats>,
ca_proto: Arc<CaProtoStats>,
insert_worker_stats: Arc<InsertWorkerStats>,
series_by_channel_stats: Arc<SeriesByChannelStats>,
ioc_finder_stats: Arc<IocFinderStats>,
@@ -157,7 +155,6 @@ impl StatsSet {
daemon,
ca_conn_set,
ca_conn,
ca_proto,
insert_worker_stats,
series_by_channel_stats,
ioc_finder_stats,
@@ -366,9 +363,8 @@ fn metrics(stats_set: &StatsSet) -> String {
let s3 = stats_set.insert_worker_stats.prometheus();
let s4 = stats_set.ca_conn.prometheus();
let s5 = stats_set.series_by_channel_stats.prometheus();
let s6 = stats_set.ca_proto.prometheus();
let s7 = stats_set.ioc_finder_stats.prometheus();
[s1, s2, s3, s4, s5, s6, s7].join("")
[s1, s2, s3, s4, s5, s7].join("")
}
pub struct RoutesResources {