daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Improve write to different retention times

Browse Source
This commit is contained in:
Dominik Werder
2024-05-09 13:33:09 +02:00
parent 4b2d648559
commit 0477504628
21 changed files with 968 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files
netfetch/src/ca/beacons.rs
+5 -5
View File
@@ -7,10 +7,10 @@ use log::*;
use netpod::ScalarType;
use netpod::Shape;
use netpod::TsNano;
use scywr::insertqueues::InsertDeques;
use scywr::iteminsertqueue::DataValue;
use scywr::iteminsertqueue::ScalarValue;
use serieswriter::writer::SeriesWriter;
use std::collections::VecDeque;
use std::io::Cursor;
use std::net::Ipv4Addr;
use std::time::SystemTime;
@@ -36,7 +36,7 @@ pub async fn listen_beacons(
sock.set_broadcast(true).unwrap();
let mut buf = Vec::new();
buf.resize(1024 * 4, 0);
let mut iqdqs = InsertDeques::new();
let mut deque = VecDeque::new();
loop {
let bb = &mut buf;
let (n, remote) = taskrun::tokio::select! {
@@ -65,12 +65,12 @@ pub async fn listen_beacons(
let ts_local = ts;
let blob = addr_u32 as i64;
let val = DataValue::Scalar(ScalarValue::I64(blob));
writer.write(ts, ts_local, val, &mut iqdqs)?;
writer.write(ts, ts_local, val, &mut deque)?;
}
}
if iqdqs.len() != 0 {
if deque.len() != 0 {
// TODO deliver to insert queue
iqdqs.clear();
deque.clear();
}
}
Ok(())
netfetch/src/ca/conn.rs
+125 -63
View File
@@ -46,9 +46,9 @@ use scywriiq::ConnectionStatusItem;
use serde::Serialize;
use series::ChannelStatusSeriesId;
use series::SeriesId;
use serieswriter::writer::EstablishWorkerJob;
use serieswriter::writer::JobId;
use serieswriter::writer::SeriesWriter;
use serieswriter::establish_worker::EstablishWorkerJob;
use serieswriter::establish_worker::JobId;
use serieswriter::rtwriter::RtWriter;
use stats::rand_xoshiro::rand_core::RngCore;
use stats::rand_xoshiro::rand_core::SeedableRng;
use stats::rand_xoshiro::Xoshiro128PlusPlus;
@@ -72,11 +72,12 @@ use taskrun::tokio;
use tokio::net::TcpStream;
const CONNECTING_TIMEOUT: Duration = Duration::from_millis(6000);
const IOC_PING_IVL: Duration = Duration::from_millis(80000);
const IOC_PING_IVL: Duration = Duration::from_millis(1000 * 80);
const DO_RATE_CHECK: bool = false;
const MONITOR_POLL_TIMEOUT: Duration = Duration::from_millis(3000);
const TIMEOUT_CHANNEL_CLOSING: Duration = Duration::from_millis(3000);
const TIMEOUT_MONITOR_PASSIVE: Duration = Duration::from_millis(3000);
const MONITOR_POLL_TIMEOUT: Duration = Duration::from_millis(6000);
const TIMEOUT_CHANNEL_CLOSING: Duration = Duration::from_millis(8000);
const TIMEOUT_MONITOR_PASSIVE: Duration = Duration::from_millis(1000 * 68);
const TIMEOUT_PONG_WAIT: Duration = Duration::from_millis(10000);
#[allow(unused)]
macro_rules! trace2 {
@@ -90,7 +91,7 @@ macro_rules! trace2 {
#[allow(unused)]
macro_rules! trace3 {
($($arg:tt)*) => {
if true {
if false {
trace!($($arg)*);
}
};
@@ -120,7 +121,7 @@ pub enum Error {
ProtocolError,
IocIssue,
Protocol(#[from] crate::ca::proto::Error),
Writer(#[from] serieswriter::writer::Error),
RtWriter(#[from] serieswriter::rtwriter::Error),
// TODO remove false positive from ThisError derive
#[allow(private_interfaces)]
UnknownCid(Cid),
@@ -137,6 +138,7 @@ pub enum Error {
Error,
DurationOutOfBounds,
NoFreeCid,
InsertQueues(#[from] scywr::insertqueues::Error),
}
impl err::ToErr for Error {
@@ -324,7 +326,7 @@ enum PollTickState {
struct WritableState {
tsbeg: Instant,
channel: CreatedState,
writer: SeriesWriter,
writer: RtWriter,
reading: ReadingState,
}
@@ -344,7 +346,10 @@ struct CreatedState {
ca_dbr_type: u16,
ca_dbr_count: u32,
ts_created: Instant,
// Updated when we receive something via monitoring or polling
ts_alive_last: Instant,
// Updated on monitoring, polling or when the channel config changes to reset the timeout
ts_activity_last: Instant,
ts_msp_last: u64,
ts_msp_grid_last: u32,
inserted_in_ts_msp: u64,
@@ -374,6 +379,7 @@ impl CreatedState {
ca_dbr_count: 0,
ts_created: tsnow,
ts_alive_last: tsnow,
ts_activity_last: tsnow,
ts_msp_last: 0,
ts_msp_grid_last: 0,
inserted_in_ts_msp: 0,
@@ -417,6 +423,12 @@ struct ChannelConf {
state: ChannelState,
}
impl ChannelConf {
pub fn poll_conf(&self) -> Option<(u64,)> {
self.conf.poll_conf()
}
}
impl ChannelState {
fn to_info(&self, cssid: ChannelStatusSeriesId, addr: SocketAddrV4, conf: ChannelConfig) -> ChannelStateInfo {
let channel_connected_info = match self {
@@ -701,6 +713,18 @@ impl CaConnEvent {
value,
}
}
pub fn desc_short(&self) -> CaConnEventDescShort {
CaConnEventDescShort {}
}
}
pub struct CaConnEventDescShort {}
impl fmt::Display for CaConnEventDescShort {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "CaConnEventDescShort {{ TODO-impl }}")
}
}
#[derive(Debug)]
@@ -781,8 +805,8 @@ pub struct CaConn {
rng: Xoshiro128PlusPlus,
writer_establish_qu: VecDeque<EstablishWorkerJob>,
writer_establish_tx: Pin<Box<SenderPolling<EstablishWorkerJob>>>,
writer_tx: Sender<(JobId, Result<SeriesWriter, serieswriter::writer::Error>)>,
writer_rx: Pin<Box<Receiver<(JobId, Result<SeriesWriter, serieswriter::writer::Error>)>>>,
writer_tx: Sender<(JobId, Result<RtWriter, serieswriter::rtwriter::Error>)>,
writer_rx: Pin<Box<Receiver<(JobId, Result<RtWriter, serieswriter::rtwriter::Error>)>>>,
tmp_ts_poll: SystemTime,
poll_tsnow: Instant,
ioid: u32,
@@ -909,9 +933,10 @@ impl CaConn {
};
self.channel_state_on_shutdown(channel_reason);
let addr = self.remote_addr_dbg.clone();
self.iqdqs
.lt_rf3_rx
.push_back(QueryItem::ConnectionStatus(ConnectionStatusItem {
// TODO handle Err:
let _ = self
.iqdqs
.emit_status_item(QueryItem::ConnectionStatus(ConnectionStatusItem {
ts: self.tmp_ts_poll,
addr,
// TODO map to appropriate status
@@ -990,7 +1015,7 @@ impl CaConn {
}
}
fn handle_writer_establish_inner(&mut self, cid: Cid, writer: SeriesWriter) -> Result<(), Error> {
fn handle_writer_establish_inner(&mut self, cid: Cid, writer: RtWriter) -> Result<(), Error> {
trace!("handle_writer_establish_inner {cid:?}");
// At this point we have created the channel and created a writer for that type and sid.
// We do not yet monitor.
@@ -999,6 +1024,8 @@ impl CaConn {
// Create a monitor for the channel.
// NOTE: must store the Writer even if not yet in Evented, we could also transition to Polled!
if let Some(conf) = self.channels.get_mut(&cid) {
// TODO refactor, should only execute this when required:
let conf_poll_conf = conf.poll_conf();
let chst = &mut conf.state;
if let ChannelState::MakingSeriesWriter(st2) = chst {
self.stats.get_series_id_ok.inc();
@@ -1008,21 +1035,21 @@ impl CaConn {
cssid: st2.channel.cssid.clone(),
status: ChannelStatus::Opened,
});
self.iqdqs.lt_rf3_rx.push_back(item);
self.iqdqs.emit_status_item(item);
}
let name = conf.conf.name();
if name.starts_with("TEST:PEAKING:") {
if let Some((ivl,)) = conf_poll_conf {
let created_state = WritableState {
tsbeg: self.poll_tsnow,
channel: std::mem::replace(&mut st2.channel, CreatedState::dummy()),
// channel: st2.channel.clone(),
writer,
reading: ReadingState::Polling(PollingState {
tsbeg: self.poll_tsnow,
poll_ivl: Duration::from_millis(1000),
poll_ivl: Duration::from_millis(ivl),
tick: PollTickState::Idle(self.poll_tsnow),
}),
};
*chst = ChannelState::Writable(created_state);
conf.state = ChannelState::Writable(created_state);
Ok(())
} else {
let subid = {
@@ -1058,7 +1085,7 @@ impl CaConn {
subid,
}),
};
*chst = ChannelState::Writable(created_state);
conf.state = ChannelState::Writable(created_state);
Ok(())
}
} else {
@@ -1175,7 +1202,7 @@ impl CaConn {
cssid: cssid.clone(),
status: ChannelStatus::Closed(channel_reason.clone()),
});
self.iqdqs.lt_rf3_rx.push_back(item);
self.iqdqs.emit_status_item(item);
*chst = ChannelState::Ended(cssid);
}
ChannelState::Error(..) => {
@@ -1307,8 +1334,8 @@ impl CaConn {
ReadingState::Monitoring(x) => {
match x.mon2state {
// actually, no differing behavior needed so far.
Monitoring2State::Passive(_) => (),
Monitoring2State::ReadPending(ioid, since) => (),
Monitoring2State::Passive(_) => {}
Monitoring2State::ReadPending(ioid, since) => {}
}
Some(x.subid.clone())
}
@@ -1350,9 +1377,10 @@ impl CaConn {
Monitoring2State::Passive(st3) => {
st3.tsbeg = tsnow;
}
Monitoring2State::ReadPending(ioid, since) => {
warn!("TODO we are waiting for a explicit caget, but received a monitor event");
st2.mon2state = Monitoring2State::Passive(Monitoring2PassiveState { tsbeg: tsnow });
Monitoring2State::ReadPending(_ioid, _since) => {
// Received EventAdd while still waiting for answer to explicit ReadNotify.
// This is fine.
self.stats.recv_event_add_while_wait_on_read_notify.inc();
}
}
let crst = &mut st.channel;
@@ -1377,9 +1405,11 @@ impl CaConn {
}
}
}
_ => {
// TODO count instead of print
error!("unexpected state: EventAddRes while having {ch_s:?}");
ChannelState::Creating(_) | ChannelState::Init(_) | ChannelState::MakingSeriesWriter(_) => {
self.stats.recv_read_notify_but_not_init_yet.inc();
}
ChannelState::Closing(_) | ChannelState::Ended(_) | ChannelState::Error(_) => {
self.stats.recv_read_notify_but_no_longer_ready.inc();
}
}
Ok(())
@@ -1479,7 +1509,7 @@ impl CaConn {
match &mut st.reading {
ReadingState::Polling(st2) => match &mut st2.tick {
PollTickState::Idle(_st3) => {
warn!("received ReadNotifyRes while in Wait state");
self.stats.recv_read_notify_while_polling_idle.inc();
}
PollTickState::Wait(st3, ioid) => {
let dt = tsnow.saturating_duration_since(*st3);
@@ -1492,21 +1522,26 @@ impl CaConn {
Self::read_notify_res_for_write(ev, st, iqdqs, stnow, tsnow, stats)?;
}
},
ReadingState::EnableMonitoring(..) => {
error!("TODO handle_read_notify_res handle EnableMonitoring");
ReadingState::EnableMonitoring(_) => {
self.stats.recv_read_notify_while_enabling_monitoring.inc();
}
ReadingState::Monitoring(st2) => match &mut st2.mon2state {
Monitoring2State::Passive(st3) => {
self.read_ioids.remove(&ioid);
if self.read_ioids.remove(&ioid).is_some() {
self.stats.recv_read_notify_state_passive_found_ioid.inc();
} else {
self.stats.recv_read_notify_state_passive.inc();
}
st3.tsbeg = tsnow;
error!("ReadNotifyRes even though we do not expect one");
}
Monitoring2State::ReadPending(ioid2, _since) => {
trace!("\nhandle_read_notify_res received ReadNotify in Monitoring2State::ReadPending\n\n");
// We don't check again for `since` here. That's done in timeout checking.
// So we could be here a little beyond timeout but we don't care about that.
if ioid != *ioid2 {
warn!("IOID mismatch ReadNotifyRes on Monitor Read Pending {ioid:?} {ioid2:?}");
// warn!("IOID mismatch ReadNotifyRes on Monitor Read Pending {ioid:?} {ioid2:?}");
self.stats.recv_read_notify_state_read_pending_bad_ioid.inc();
} else {
self.stats.recv_read_notify_state_read_pending.inc();
}
self.read_ioids.remove(&ioid);
st2.mon2state = Monitoring2State::Passive(Monitoring2PassiveState { tsbeg: tsnow });
@@ -1550,7 +1585,7 @@ impl CaConn {
payload_len: u32,
value: CaEventValue,
crst: &mut CreatedState,
writer: &mut SeriesWriter,
writer: &mut RtWriter,
iqdqs: &mut InsertDeques,
tsnow: Instant,
stnow: SystemTime,
@@ -1558,6 +1593,7 @@ impl CaConn {
) -> Result<(), Error> {
// debug!("event_add_ingest payload_len {} value {:?}", payload_len, value);
crst.ts_alive_last = tsnow;
crst.ts_activity_last = tsnow;
crst.item_recv_ivl_ema.tick(tsnow);
crst.recv_count += 1;
crst.recv_bytes += payload_len as u64;
@@ -1578,7 +1614,7 @@ impl CaConn {
crst.muted_before = 0;
crst.insert_item_ivl_ema.tick(tsnow);
}
Self::check_ev_value_data(&value.data, writer.scalar_type())?;
Self::check_ev_value_data(&value.data, &writer.scalar_type())?;
{
let val: DataValue = value.data.into();
writer.write(TsNano::from_ns(ts), TsNano::from_ns(ts_local), val, iqdqs)?;
@@ -1849,12 +1885,12 @@ impl CaConn {
Ok(())
}
fn check_channels_alive(&mut self, tsnow: Instant, cx: &mut Context) -> Result<(), Error> {
trace2!("check_channels_alive {addr:?}", addr = &self.remote_addr_dbg);
fn check_channels_alive(&mut self, tsnow: Instant, _cx: &mut Context) -> Result<(), Error> {
trace3!("check_channels_alive {}", self.remote_addr_dbg);
if let Some(started) = self.ioc_ping_start {
if started + Duration::from_millis(4000) < tsnow {
if started + TIMEOUT_PONG_WAIT < tsnow {
self.stats.pong_timeout().inc();
warn!("pong timeout {addr:?}", addr = self.remote_addr_dbg);
warn!("pong timeout {}", self.remote_addr_dbg);
self.ioc_ping_start = None;
let item = CaConnEvent {
ts: tsnow,
@@ -1867,7 +1903,6 @@ impl CaConn {
if self.ioc_ping_next < tsnow {
if let Some(proto) = &mut self.proto {
self.stats.ping_start().inc();
info!("start ping");
self.ioc_ping_start = Some(tsnow);
let msg = CaMsg::from_ty_ts(CaMsgTy::Echo, tsnow);
proto.push_out(msg);
@@ -1889,8 +1924,8 @@ impl CaConn {
// TODO handle timeout check
}
ReadingState::Monitoring(st3) => match &st3.mon2state {
Monitoring2State::Passive(st4) => {}
Monitoring2State::ReadPending(_, tsbeg) => {
Monitoring2State::Passive(_st4) => {}
Monitoring2State::ReadPending(_, _) => {
// This is handled in check_channels_state_poll
// TODO should unify.
}
@@ -1898,14 +1933,14 @@ impl CaConn {
ReadingState::StopMonitoringForPolling(_) => {
// TODO handle timeout check
}
ReadingState::Polling(st3) => {
ReadingState::Polling(_st3) => {
// This is handled in check_channels_state_poll
// TODO should unify.
}
}
if tsnow.duration_since(st2.channel.ts_alive_last) >= Duration::from_millis(10000) {
warn!("TODO assume channel not alive because nothing received, but should do CAGET");
if st2.channel.ts_activity_last + conf.conf.expect_activity_within() < tsnow {
not_alive_count += 1;
self.stats.channel_not_alive_no_activity.inc();
} else {
alive_count += 1;
}
@@ -2082,6 +2117,7 @@ impl CaConn {
ca_dbr_count: k.data_count,
ts_created: tsnow,
ts_alive_last: tsnow,
ts_activity_last: tsnow,
ts_msp_last: 0,
ts_msp_grid_last: 0,
inserted_in_ts_msp: u64::MAX,
@@ -2104,8 +2140,10 @@ impl CaConn {
JobId(cid.0 as _),
self.backend.clone(),
conf.conf.name().into(),
cssid,
scalar_type,
shape,
conf.conf.min_quiets(),
self.writer_tx.clone(),
self.tmp_ts_poll,
);
@@ -2149,12 +2187,11 @@ impl CaConn {
self.stats.tcp_connected.inc();
let addr = addr.clone();
self.iqdqs
.lt_rf3_rx
.push_back(QueryItem::ConnectionStatus(ConnectionStatusItem {
.emit_status_item(QueryItem::ConnectionStatus(ConnectionStatusItem {
ts: self.tmp_ts_poll,
addr,
status: ConnectionStatus::Established,
}));
}))?;
self.backoff_reset();
let proto = CaProto::new(
tcp,
@@ -2167,29 +2204,27 @@ impl CaConn {
Ok(Ready(Some(())))
}
Ok(Err(e)) => {
debug!("error connect to {addr} {e}");
info!("error connect to {addr} {e}");
let addr = addr.clone();
self.iqdqs
.lt_rf3_rx
.push_back(QueryItem::ConnectionStatus(ConnectionStatusItem {
.emit_status_item(QueryItem::ConnectionStatus(ConnectionStatusItem {
ts: self.tmp_ts_poll,
addr,
status: ConnectionStatus::ConnectError,
}));
}))?;
self.trigger_shutdown(ShutdownReason::IoError);
Ok(Ready(Some(())))
}
Err(e) => {
// TODO log with exponential backoff
debug!("timeout connect to {addr} {e}");
info!("timeout connect to {addr} {e}");
let addr = addr.clone();
self.iqdqs
.lt_rf3_rx
.push_back(QueryItem::ConnectionStatus(ConnectionStatusItem {
.emit_status_item(QueryItem::ConnectionStatus(ConnectionStatusItem {
ts: self.tmp_ts_poll,
addr,
status: ConnectionStatus::ConnectTimeout,
}));
}))?;
self.trigger_shutdown(ShutdownReason::IocTimeout);
Ok(Ready(Some(())))
}
@@ -2372,7 +2407,7 @@ impl CaConn {
count,
bytes,
});
self.iqdqs.lt_rf3_rx.push_back(item);
self.iqdqs.emit_status_item(item)?;
}
}
}
@@ -2454,7 +2489,10 @@ impl CaConn {
use scywr::senderpolling::Error as SpErr;
match e {
SpErr::NoSendInProgress => return Err(Error::NotSending),
SpErr::Closed(_) => return Err(Error::ClosedSending),
SpErr::Closed(_) => {
error!("{self_name} queue closed id {:10}", id);
return Err(Error::ClosedSending);
}
}
}
Pending => {
@@ -2471,6 +2509,11 @@ impl CaConn {
Ok(Ready(None))
}
}
fn log_queues_summary(&self) {
self.iqdqs.log_summary();
self.iqsp.log_summary();
}
}
// $have is tuple (have_progress, have_pending))
@@ -2592,6 +2635,7 @@ impl Stream for CaConn {
cx,
stats_fn
);
let stats2 = self.stats.clone();
let stats_fn = move |item: &VecDeque<QueryItem>| {
stats2.iiq_batch_len().ingest(item.len() as u32);
@@ -2607,6 +2651,7 @@ impl Stream for CaConn {
cx,
stats_fn
);
let stats2 = self.stats.clone();
let stats_fn = move |item: &VecDeque<QueryItem>| {
stats2.iiq_batch_len().ingest(item.len() as u32);
@@ -2622,6 +2667,22 @@ impl Stream for CaConn {
cx,
stats_fn
);
let stats2 = self.stats.clone();
let stats_fn = move |item: &VecDeque<QueryItem>| {
stats2.iiq_batch_len().ingest(item.len() as u32);
};
flush_queue_dqs!(
self,
lt_rf3_rx,
lt_rf3_sp_pin,
send_batched::<256, _>,
32,
(&mut have_progress, &mut have_pending),
"lt_rf3_rx",
cx,
stats_fn
);
}
let lts3 = Instant::now();
@@ -2728,6 +2789,7 @@ impl Stream for CaConn {
continue;
} else if have_pending {
debug!("is_shutdown NOT queues_out_flushed pend {}", self.remote_addr_dbg);
self.log_queues_summary();
self.stats.poll_pending().inc();
Pending
} else {
netfetch/src/ca/connset.rs
+2 -2
View File
@@ -40,7 +40,6 @@ use scywr::iteminsertqueue::QueryItem;
use scywr::senderpolling::SenderPolling;
use serde::Serialize;
use series::ChannelStatusSeriesId;
use serieswriter::writer::EstablishWorkerJob;
use statemap::ActiveChannelState;
use statemap::CaConnStateValue;
use statemap::ChannelState;
@@ -64,6 +63,7 @@ use std::pin::Pin;
use netpod::OnDrop;
use scywr::insertqueues::InsertQueuesTx;
use serieswriter::establish_worker::EstablishWorkerJob;
use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
@@ -1102,7 +1102,7 @@ impl CaConnSet {
) -> Result<EndOfStreamReason, Error> {
let mut eos_reason = None;
while let Some(item) = conn.next().await {
trace!("ca_conn_item_merge_inner item {item:?}");
trace!("ca_conn_item_merge_inner item {}", item.desc_short());
if let Some(x) = eos_reason {
let e = Error::with_msg_no_trace(format!("CaConn delivered already eos {addr} {x:?}"));
error!("{e}");
netfetch/src/ca/findioc.rs
+4 -4
View File
@@ -603,17 +603,17 @@ impl Stream for FindIocStream {
have_progress = true;
}
Ready(Err(e)) => {
error!("{e:?}");
error!("{e}");
}
Pending => {
g.clear_ready();
warn!("socket seemed ready for write, but is not");
// warn!("socket seemed ready for write, but is not");
have_progress = true;
}
},
Ready(Err(e)) => {
let e = Error::with_msg_no_trace(format!("{e:?}"));
error!("poll_write_ready {e:?}");
error!("poll_write_ready {e}");
let e = Error::from_string(e);
}
Pending => {}
}
netfetch/src/ca/proto.rs
+35 -33
View File
@@ -788,15 +788,7 @@ impl CaMsg {
};
CaMsg::from_ty_ts(CaMsgTy::Error(e), tsnow)
}
20 => {
let name = std::ffi::CString::new(payload)
.map(|s| s.into_string().unwrap_or_else(|e| format!("{e:?}")))
.unwrap_or_else(|e| format!("{e:?}"));
CaMsg::from_ty_ts(CaMsgTy::ClientNameRes(ClientNameRes { name }), tsnow)
}
// TODO make response type for host name:
21 => CaMsg::from_ty_ts(CaMsgTy::HostName("TODOx5288".into()), tsnow),
6 => {
0x06 => {
if hi.payload_len() != 8 {
warn!("protocol error: search result is expected with fixed payload size 8");
}
@@ -815,29 +807,6 @@ impl CaMsg {
});
CaMsg::from_ty_ts(ty, tsnow)
}
18 => {
let ty = CaMsgTy::CreateChanRes(CreateChanRes {
data_type: hi.data_type,
// TODO what am I supposed to use here in case of extended header?
data_count: hi.data_count() as _,
cid: hi.param1,
sid: hi.param2,
});
CaMsg::from_ty_ts(ty, tsnow)
}
22 => {
// TODO use different structs for request and response:
let ty = CaMsgTy::AccessRightsRes(AccessRightsRes {
cid: hi.param1,
rights: hi.param2,
});
CaMsg::from_ty_ts(ty, tsnow)
}
26 => {
// TODO use different structs for request and response:
let ty = CaMsgTy::CreateChanFail(CreateChanFail { cid: hi.param1 });
CaMsg::from_ty_ts(ty, tsnow)
}
0x01 => {
if payload.len() < 12 {
if payload.len() == 0 {
@@ -897,7 +866,40 @@ impl CaMsg {
});
CaMsg::from_ty_ts(ty, tsnow)
}
0x11 => CaMsg::from_ty_ts(CaMsgTy::Echo, tsnow),
0x12 => {
let ty = CaMsgTy::CreateChanRes(CreateChanRes {
data_type: hi.data_type,
// TODO what am I supposed to use here in case of extended header?
data_count: hi.data_count() as _,
cid: hi.param1,
sid: hi.param2,
});
CaMsg::from_ty_ts(ty, tsnow)
}
0x16 => {
let ty = CaMsgTy::AccessRightsRes(AccessRightsRes {
cid: hi.param1,
rights: hi.param2,
});
CaMsg::from_ty_ts(ty, tsnow)
}
0x17 => {
let ty = CaMsgTy::Echo;
CaMsg::from_ty_ts(ty, tsnow)
}
0x1a => {
// TODO use different structs for request and response:
let ty = CaMsgTy::CreateChanFail(CreateChanFail { cid: hi.param1 });
CaMsg::from_ty_ts(ty, tsnow)
}
0x14 => {
let name = std::ffi::CString::new(payload)
.map(|s| s.into_string().unwrap_or_else(|e| format!("{e:?}")))
.unwrap_or_else(|e| format!("{e:?}"));
CaMsg::from_ty_ts(CaMsgTy::ClientNameRes(ClientNameRes { name }), tsnow)
}
// TODO make response type for host name:
0x15 => CaMsg::from_ty_ts(CaMsgTy::HostName("TODOx5288".into()), tsnow),
x => return Err(Error::CaCommandNotSupported(x)),
};
Ok(msg)
netfetch/src/conf.rs
+80 -2
View File
@@ -5,6 +5,7 @@ use regex::Regex;
use scywr::config::ScyllaIngestConfig;
use serde::Deserialize;
use serde::Serialize;
use serieswriter::rtwriter::MinQuiets;
use std::collections::BTreeMap;
use std::path::Path;
use std::path::PathBuf;
@@ -39,6 +40,8 @@ pub struct CaIngestOpts {
insert_frac: Option<u64>,
use_rate_limit_queue: Option<bool>,
pub test_bsread_addr: Option<String>,
#[serde(default)]
scylla_disable: bool,
}
impl CaIngestOpts {
@@ -109,12 +112,16 @@ impl CaIngestOpts {
pub fn use_rate_limit_queue(&self) -> bool {
self.use_rate_limit_queue.unwrap_or(false)
}
pub fn scylla_disable(&self) -> bool {
self.scylla_disable
}
}
#[test]
fn parse_config_minimal() {
let conf = r###"
backend: scylla
backend: test_backend
timeout: 10m 3s 45ms
api_bind: "0.0.0.0:3011"
channels: /some/path/file.txt
@@ -127,7 +134,7 @@ postgresql:
user: USER
pass: PASS
name: NAME
scylla:
scylla_st:
hosts:
- sf-nube-11:19042
- sf-nube-12:19042
@@ -525,4 +532,75 @@ impl ChannelConfig {
pub fn name(&self) -> &str {
&self.name
}
pub fn is_polled(&self) -> bool {
self.arch.is_polled
}
pub fn poll_conf(&self) -> Option<(u64,)> {
if self.is_polled() {
if let Some(ChannelReadConfig::Poll(x)) = self.arch.short_term {
Some((x.as_millis() as u64,))
} else if let Some(ChannelReadConfig::Poll(x)) = self.arch.medium_term {
Some((x.as_millis() as u64,))
} else if let Some(ChannelReadConfig::Poll(x)) = self.arch.long_term {
Some((x.as_millis() as u64,))
} else {
Some((60,))
}
} else {
None
}
}
/// Only used when in monitoring mode. If we do not see activity for this Duration then
/// we issue a manual read to see if the channel is alive.
pub fn manual_poll_on_quiet(&self) -> Duration {
Duration::from_secs(120)
}
pub fn expect_activity_within(&self) -> Duration {
let dur = if self.is_polled() {
// It would be anyway invalid to be polled and specify a monitor record policy.
match self.arch.short_term {
Some(ChannelReadConfig::Poll(x)) => x,
Some(ChannelReadConfig::Monitor) => self.manual_poll_on_quiet(),
None => match self.arch.medium_term {
Some(ChannelReadConfig::Poll(x)) => x,
Some(ChannelReadConfig::Monitor) => self.manual_poll_on_quiet(),
None => match self.arch.long_term {
Some(ChannelReadConfig::Poll(x)) => x,
Some(ChannelReadConfig::Monitor) => self.manual_poll_on_quiet(),
None => {
// This is an invalid configuration, so just a fallback
self.manual_poll_on_quiet()
}
},
},
}
} else {
self.manual_poll_on_quiet()
};
dur + Duration::from_millis(1000 * 10)
}
pub fn min_quiets(&self) -> MinQuiets {
MinQuiets {
st: match self.arch.short_term {
Some(ChannelReadConfig::Monitor) => Duration::ZERO,
Some(ChannelReadConfig::Poll(x)) => x,
None => Duration::MAX,
},
mt: match self.arch.medium_term {
Some(ChannelReadConfig::Monitor) => Duration::ZERO,
Some(ChannelReadConfig::Poll(x)) => x,
None => Duration::MAX,
},
lt: match self.arch.long_term {
Some(ChannelReadConfig::Monitor) => Duration::ZERO,
Some(ChannelReadConfig::Poll(x)) => x,
None => Duration::MAX,
},
}
}
}
netfetch/src/metrics/postingest.rs
+10 -7
View File
@@ -55,10 +55,11 @@ pub async fn process_api_query_items(
#[allow(irrefutable_let_patterns)]
while let item = taskrun::tokio::time::timeout(Duration::from_millis(500), item_rx.recv()).await {
let deque = &mut iqdqs.st_rf3_rx;
let tsnow = Instant::now();
if tsnow.saturating_duration_since(sw_tick_last) >= Duration::from_millis(5000) {
sw_tick_last = tsnow;
tick_writers(mucache.all_ref_mut(), &mut iqdqs)?;
tick_writers(mucache.all_ref_mut(), deque)?;
}
let item = match item {
Ok(Ok(item)) => item,
@@ -83,23 +84,25 @@ pub async fn process_api_query_items(
stnow,
)
.await?;
sw.write(item.ts, item.ts, item.val, &mut iqdqs)?;
sw.write(item.ts, item.ts, item.val, deque)?;
iqtx.send_all(&mut iqdqs).await.map_err(|_| Error::SendError)?;
}
finish_writers(mucache.all_ref_mut(), &mut iqdqs)?;
let deque = &mut iqdqs.st_rf3_rx;
finish_writers(mucache.all_ref_mut(), deque)?;
iqtx.send_all(&mut iqdqs).await.map_err(|_| Error::SendError)?;
Ok(())
}
fn tick_writers(sws: Vec<&mut SeriesWriter>, iqdqs: &mut InsertDeques) -> Result<(), Error> {
fn tick_writers(sws: Vec<&mut SeriesWriter>, deque: &mut VecDeque<QueryItem>) -> Result<(), Error> {
for sw in sws {
sw.tick(iqdqs)?;
sw.tick(deque)?;
}
Ok(())
}
fn finish_writers(sws: Vec<&mut SeriesWriter>, iqdqs: &mut InsertDeques) -> Result<(), Error> {
fn finish_writers(sws: Vec<&mut SeriesWriter>, deque: &mut VecDeque<QueryItem>) -> Result<(), Error> {
for sw in sws {
sw.tick(iqdqs)?;
sw.tick(deque)?;
}
Ok(())
}