daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Refactor config from st, mt, lt

Browse Source
This commit is contained in:
Dominik Werder
2024-05-02 11:51:42 +02:00
parent c98c638381
commit 75f4ba58e1
12 changed files with 290 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
netfetch/src/ca/beacons.rs
+36 -4
View File
@@ -1,21 +1,42 @@
use async_channel::Sender;
use bytes::Buf;
use dbpg::seriesbychannel::ChannelInfoQuery;
use err::thiserror;
use err::ThisError;
use log::*;
use netpod::ScalarType;
use netpod::Shape;
use netpod::TsNano;
use scywr::iteminsertqueue::DataValue;
use scywr::iteminsertqueue::ScalarValue;
use serieswriter::writer::SeriesWriter;
use std::collections::VecDeque;
use std::io::Cursor;
use std::net::Ipv4Addr;
use std::time::SystemTime;
use taskrun::tokio::net::UdpSocket;
#[derive(Debug, ThisError)]
pub enum Error {
Io(#[from] std::io::Error),
SeriesWriter(#[from] serieswriter::writer::Error),
}
pub async fn listen_beacons(mut cancel: taskrun::tokio::sync::mpsc::Receiver<u32>) -> Result<(), Error> {
pub async fn listen_beacons(
mut cancel: taskrun::tokio::sync::mpsc::Receiver<u32>,
worker_tx: Sender<ChannelInfoQuery>,
backend: String,
) -> Result<(), Error> {
let stnow = SystemTime::now();
let channel = "epics-ca-beacons".to_string();
let scalar_type = ScalarType::U64;
let shape = Shape::Scalar;
let mut writer = SeriesWriter::establish(worker_tx, backend, channel, scalar_type, shape, stnow).await?;
let sock = UdpSocket::bind("0.0.0.0:5065").await?;
sock.set_broadcast(true).unwrap();
let mut buf = Vec::new();
buf.resize(1024 * 4, 0);
let mut item_qu = VecDeque::new();
loop {
let bb = &mut buf;
let (n, remote) = taskrun::tokio::select! {
@@ -34,12 +55,23 @@ pub async fn listen_beacons(mut cancel: taskrun::tokio::sync::mpsc::Receiver<u32
let ver = cur.get_u16();
let port = cur.get_u16();
let _seqid = cur.get_u32();
let addr = cur.get_u32();
let addr = Ipv4Addr::from(addr);
let addr_u32 = cur.get_u32();
let addr = Ipv4Addr::from(addr_u32);
if cmd == 0x0d {
debug!("beacon {remote} {ver} {addr} {port}")
debug!("beacon {remote} {ver} {addr} {port}");
let stnow = SystemTime::now();
let x = stnow.duration_since(SystemTime::UNIX_EPOCH).unwrap();
let ts = TsNano::from_ms(1000 * x.as_secs() + x.subsec_millis() as u64);
let ts_local = ts;
let blob = addr_u32 as i64;
let val = DataValue::Scalar(ScalarValue::I64(blob));
writer.write(ts, ts_local, val, &mut item_qu)?;
}
}
if item_qu.len() != 0 {
// TODO deliver to insert queue
item_qu.clear();
}
}
Ok(())
}
netfetch/src/ca/conn.rs
+17 -8
View File
@@ -96,12 +96,21 @@ macro_rules! trace3 {
#[allow(unused)]
macro_rules! trace4 {
($($arg:tt)*) => {
if true {
if false {
trace!($($arg)*);
}
};
}
#[allow(unused)]
macro_rules! trace_flush_queue {
($($arg:tt)*) => {
if false {
trace3!($($arg)*);
}
};
}
#[derive(Debug, ThisError)]
pub enum Error {
NoProtocol,
@@ -1932,7 +1941,7 @@ impl CaConn {
CaMsgTy::SearchRes(k) => {
let a = k.addr.to_be_bytes();
let addr = format!("{}.{}.{}.{}:{}", a[0], a[1], a[2], a[3], k.tcp_port);
trace!("Search result indicates server address: {addr}");
trace!("search result indicates server address: {addr}");
// TODO count this unexpected case.
}
CaMsgTy::CreateChanRes(k) => {
@@ -1940,12 +1949,12 @@ impl CaConn {
cx.waker().wake_by_ref();
}
CaMsgTy::EventAddRes(ev) => {
trace2!("got EventAddRes {:?} cnt {}", camsg.ts, ev.data_count);
trace4!("got EventAddRes {:?} cnt {}", camsg.ts, ev.data_count);
self.stats.event_add_res_recv.inc();
Self::handle_event_add_res(self, ev, tsnow)?
}
CaMsgTy::EventAddResEmpty(ev) => {
trace2!("got EventAddResEmpty {:?}", camsg.ts);
trace4!("got EventAddResEmpty {:?}", camsg.ts);
Self::handle_event_add_res_empty(self, ev, tsnow)?
}
CaMsgTy::ReadNotifyRes(ev) => Self::handle_read_notify_res(self, ev, tsnow)?,
@@ -1957,7 +1966,7 @@ impl CaConn {
self.stats.pong_recv_lat().ingest(dt);
} else {
let addr = &self.remote_addr_dbg;
warn!("Received Echo even though we didn't asked for it {addr:?}");
warn!("received Echo even though we didn't asked for it {addr:?}");
}
self.ioc_ping_last = tsnow;
self.ioc_ping_next = tsnow + Self::ioc_ping_ivl_rng(&mut self.rng);
@@ -2144,7 +2153,7 @@ impl CaConn {
self.backoff_reset();
let proto = CaProto::new(
tcp,
self.remote_addr_dbg.clone(),
self.remote_addr_dbg.to_string(),
self.opts.array_truncate,
self.ca_proto_stats.clone(),
);
@@ -2408,7 +2417,7 @@ impl CaConn {
{
use Poll::*;
if qu.len() != 0 {
trace3!("attempt_flush_queue id {:7} len {}", id, qu.len());
trace_flush_queue!("attempt_flush_queue id {:7} len {}", id, qu.len());
}
let mut have_progress = false;
let mut i = 0;
@@ -2431,7 +2440,7 @@ impl CaConn {
if sp.is_sending() {
match sp.poll_unpin(cx) {
Ready(Ok(())) => {
trace3!("attempt_flush_queue id {:7} send done", id);
trace_flush_queue!("attempt_flush_queue id {:7} send done", id);
have_progress = true;
}
Ready(Err(e)) => {
netfetch/src/ca/connset.rs
+23 -16
View File
@@ -63,6 +63,7 @@ use std::net::SocketAddrV4;
use std::pin::Pin;
use netpod::OnDrop;
use scywr::insertqueues::InsertQueuesTx;
use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
@@ -367,7 +368,7 @@ pub struct CaConnSet {
find_ioc_query_queue: VecDeque<IocAddrQuery>,
find_ioc_query_sender: Pin<Box<SenderPolling<IocAddrQuery>>>,
find_ioc_res_rx: Pin<Box<Receiver<VecDeque<FindIocRes>>>>,
storage_insert_tx: Pin<Box<Sender<VecDeque<QueryItem>>>>,
iqtx: Pin<Box<InsertQueuesTx>>,
storage_insert_queue: VecDeque<VecDeque<QueryItem>>,
storage_insert_sender: Pin<Box<SenderPolling<VecDeque<QueryItem>>>>,
ca_conn_res_tx: Pin<Box<Sender<(SocketAddr, CaConnEvent)>>>,
@@ -398,7 +399,7 @@ impl CaConnSet {
pub fn start(
backend: String,
local_epics_hostname: String,
storage_insert_tx: Sender<VecDeque<QueryItem>>,
iqtx: InsertQueuesTx,
channel_info_query_tx: Sender<ChannelInfoQuery>,
ingest_opts: CaIngestOpts,
establish_worker_tx: async_channel::Sender<EstablishWorkerJob>,
@@ -435,9 +436,12 @@ impl CaConnSet {
find_ioc_query_queue: VecDeque::new(),
find_ioc_query_sender: Box::pin(SenderPolling::new(find_ioc_query_tx)),
find_ioc_res_rx: Box::pin(find_ioc_res_rx),
storage_insert_tx: Box::pin(storage_insert_tx.clone()),
iqtx: Box::pin(iqtx.clone()),
storage_insert_queue: VecDeque::new(),
storage_insert_sender: Box::pin(SenderPolling::new(storage_insert_tx)),
// TODO simplify for all combinations
storage_insert_sender: Box::pin(SenderPolling::new(iqtx.st_rf3_tx.clone())),
ca_conn_res_tx: Box::pin(ca_conn_res_tx),
ca_conn_res_rx: Box::pin(ca_conn_res_rx),
shutdown_stopping: false,
@@ -480,13 +484,17 @@ impl CaConnSet {
async fn run(mut this: CaConnSet) -> Result<(), Error> {
trace!("CaConnSet run begin");
let (beacons_cancel_guard_tx, rx) = taskrun::tokio::sync::mpsc::channel(12);
let beacons_jh = tokio::spawn(async move {
if false {
crate::ca::beacons::listen_beacons(rx).await
} else {
Ok(())
}
});
let beacons_jh = {
let tx2 = this.channel_info_query_tx.clone().unwrap();
let backend = this.backend.clone();
tokio::spawn(async move {
if false {
crate::ca::beacons::listen_beacons(rx, tx2, backend).await
} else {
Ok(())
}
})
};
let _g_beacon = OnDrop::new(move || {});
loop {
let x = this.next().await;
@@ -1039,7 +1047,7 @@ impl CaConnSet {
add.backend.clone(),
addr_v4,
self.local_epics_hostname.clone(),
self.storage_insert_tx.as_ref().get_ref().clone(),
self.iqtx.st_rf3_tx.clone(),
self.channel_info_query_tx
.clone()
.ok_or_else(|| Error::with_msg_no_trace("no more channel_info_query_tx available"))?,
@@ -1050,8 +1058,7 @@ impl CaConnSet {
let conn_tx = conn.conn_command_tx();
let conn_stats = conn.stats();
let tx1 = self.ca_conn_res_tx.as_ref().get_ref().clone();
let tx2 = self.storage_insert_tx.as_ref().get_ref().clone();
let jh = tokio::spawn(Self::ca_conn_item_merge(conn, tx1, tx2, addr, self.stats.clone()));
let jh = tokio::spawn(Self::ca_conn_item_merge(conn, tx1, addr, self.stats.clone()));
let ca_conn_res = CaConnRes {
state: CaConnState::new(CaConnStateValue::Fresh),
sender: Box::pin(conn_tx.into()),
@@ -1065,7 +1072,6 @@ impl CaConnSet {
async fn ca_conn_item_merge(
conn: CaConn,
tx1: Sender<(SocketAddr, CaConnEvent)>,
_tx2: Sender<VecDeque<QueryItem>>,
addr: SocketAddr,
stats: Arc<CaConnSetStats>,
) -> Result<(), Error> {
@@ -1546,7 +1552,8 @@ impl Stream for CaConnSet {
trace4!("CaConnSet poll loop");
self.stats.poll_loop_begin().inc();
self.stats.storage_insert_tx_len.set(self.storage_insert_tx.len() as _);
// TODO generalize to all combinations
self.stats.storage_insert_tx_len.set(self.iqtx.st_rf3_tx.len() as _);
self.stats
.storage_insert_queue_len
.set(self.storage_insert_queue.len() as _);
netfetch/src/ca/proto.rs
+4 -5
View File
@@ -8,7 +8,6 @@ use slidebuf::SlideBuf;
use stats::CaProtoStats;
use std::collections::VecDeque;
use std::io;
use std::net::SocketAddrV4;
use std::pin::Pin;
use std::sync::Arc;
use std::task::Context;
@@ -1045,7 +1044,7 @@ impl CaState {
pub struct CaProto {
tcp: TcpStream,
tcp_eof: bool,
remote_addr_dbg: SocketAddrV4,
remote_name: String,
state: CaState,
buf: SlideBuf,
outbuf: SlideBuf,
@@ -1058,11 +1057,11 @@ pub struct CaProto {
}
impl CaProto {
pub fn new(tcp: TcpStream, remote_addr_dbg: SocketAddrV4, array_truncate: usize, stats: Arc<CaProtoStats>) -> Self {
pub fn new(tcp: TcpStream, remote_name: String, array_truncate: usize, stats: Arc<CaProtoStats>) -> Self {
Self {
tcp,
tcp_eof: false,
remote_addr_dbg,
remote_name,
state: CaState::StdHead,
buf: SlideBuf::new(PROTO_INPUT_BUF_CAP as usize),
outbuf: SlideBuf::new(1024 * 256),
@@ -1186,7 +1185,7 @@ impl CaProto {
debug!(
"peer done {:?} {:?} {:?}",
self.tcp.peer_addr(),
self.remote_addr_dbg,
self.remote_name,
self.state
);
self.tcp_eof = true;
netfetch/src/conf.rs
+13 -12
View File
@@ -27,11 +27,9 @@ pub struct CaIngestOpts {
#[serde(default, with = "humantime_serde")]
timeout: Option<Duration>,
postgresql: Database,
scylla: ScyllaIngestConfig,
#[serde(default)]
scylla_mt: Option<ScyllaIngestConfig>,
#[serde(default)]
scylla_lt: Option<ScyllaIngestConfig>,
scylla_st: ScyllaIngestConfig,
scylla_mt: ScyllaIngestConfig,
scylla_lt: ScyllaIngestConfig,
array_truncate: Option<u64>,
insert_worker_count: Option<usize>,
insert_worker_concurrency: Option<usize>,
@@ -56,16 +54,16 @@ impl CaIngestOpts {
&self.postgresql
}
pub fn scylla_config(&self) -> &ScyllaIngestConfig {
&self.scylla
pub fn scylla_config_st(&self) -> &ScyllaIngestConfig {
&self.scylla_st
}
pub fn scylla_config_mt(&self) -> Option<&ScyllaIngestConfig> {
self.scylla_mt.as_ref()
pub fn scylla_config_mt(&self) -> &ScyllaIngestConfig {
&self.scylla_mt
}
pub fn scylla_config_lt(&self) -> Option<&ScyllaIngestConfig> {
self.scylla_lt.as_ref()
pub fn scylla_config_lt(&self) -> &ScyllaIngestConfig {
&self.scylla_lt
}
pub fn search(&self) -> &Vec<String> {
@@ -140,7 +138,10 @@ scylla:
assert_eq!(conf.channels, Some(PathBuf::from("/some/path/file.txt")));
assert_eq!(&conf.api_bind, "0.0.0.0:3011");
assert_eq!(conf.search.get(0), Some(&"172.26.0.255".to_string()));
assert_eq!(conf.scylla.hosts().get(1), Some(&"sf-nube-12:19042".to_string()));
assert_eq!(
conf.scylla_config_st().hosts().get(1),
Some(&"sf-nube-12:19042".to_string())
);
assert_eq!(conf.timeout, Some(Duration::from_millis(1000 * (60 * 10 + 3) + 45)));
}