843 lines
30 KiB
Rust
843 lines
30 KiB
Rust
pub mod conn;
|
|
pub mod findioc;
|
|
pub mod proto;
|
|
pub mod search;
|
|
pub mod store;
|
|
|
|
use self::store::DataStore;
|
|
use crate::ca::conn::ConnCommand;
|
|
use crate::errconv::ErrConv;
|
|
use crate::store::{CommonInsertItemQueue, QueryItem};
|
|
use async_channel::Sender;
|
|
use conn::CaConn;
|
|
use err::Error;
|
|
use futures_util::future::Fuse;
|
|
use futures_util::stream::FuturesUnordered;
|
|
use futures_util::{FutureExt, StreamExt};
|
|
use log::*;
|
|
use netpod::{Database, ScyllaConfig};
|
|
use serde::{Deserialize, Serialize};
|
|
use stats::{CaConnStats, CaConnStatsAgg, CaConnStatsAggDiff};
|
|
use std::collections::{BTreeMap, VecDeque};
|
|
use std::ffi::CStr;
|
|
use std::mem::MaybeUninit;
|
|
use std::net::{IpAddr, Ipv4Addr, SocketAddrV4};
|
|
use std::path::PathBuf;
|
|
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
|
use std::sync::{Arc, Mutex, Once};
|
|
use std::time::Duration;
|
|
use tokio::fs::OpenOptions;
|
|
use tokio::io::AsyncReadExt;
|
|
use tokio::sync::Mutex as TokMx;
|
|
use tokio::task::JoinHandle;
|
|
use tokio_postgres::Client as PgClient;
|
|
|
|
static mut METRICS: Option<Mutex<Option<CaConnStatsAgg>>> = None;
|
|
static METRICS_ONCE: Once = Once::new();
|
|
|
|
pub fn get_metrics() -> &'static mut Option<CaConnStatsAgg> {
|
|
METRICS_ONCE.call_once(|| unsafe {
|
|
METRICS = Some(Mutex::new(None));
|
|
});
|
|
let mut g = unsafe { METRICS.as_mut().unwrap().lock().unwrap() };
|
|
let ret: &mut Option<CaConnStatsAgg> = &mut *g;
|
|
let ret = unsafe { &mut *(ret as *mut _) };
|
|
ret
|
|
}
|
|
|
|
#[derive(Debug, Serialize, Deserialize)]
|
|
struct ChannelConfig {
|
|
backend: String,
|
|
channels: Vec<String>,
|
|
search: Vec<String>,
|
|
#[serde(default)]
|
|
search_blacklist: Vec<String>,
|
|
#[serde(default)]
|
|
tmp_remove: Vec<String>,
|
|
addr_bind: Option<IpAddr>,
|
|
addr_conn: Option<IpAddr>,
|
|
whitelist: Option<String>,
|
|
blacklist: Option<String>,
|
|
max_simul: Option<usize>,
|
|
timeout: Option<u64>,
|
|
postgresql: Database,
|
|
scylla: ScyllaConfig,
|
|
array_truncate: Option<usize>,
|
|
insert_worker_count: Option<usize>,
|
|
insert_scylla_sessions: Option<usize>,
|
|
insert_queue_max: Option<usize>,
|
|
insert_item_queue_cap: Option<usize>,
|
|
api_bind: Option<String>,
|
|
local_epics_hostname: Option<String>,
|
|
}
|
|
|
|
#[test]
|
|
fn parse_config_minimal() {
|
|
let conf = r###"
|
|
backend: scylla
|
|
api_bind: 0.0.0.0:3011
|
|
channels:
|
|
- CHANNEL-1:A
|
|
- CHANNEL-1:B
|
|
- CHANNEL-2:A
|
|
search:
|
|
- 172.26.0.255
|
|
- 172.26.2.255
|
|
postgresql:
|
|
host: host.example.com
|
|
port: 5432
|
|
user: USER
|
|
pass: PASS
|
|
name: NAME
|
|
scylla:
|
|
hosts:
|
|
- sf-nube-11:19042
|
|
- sf-nube-12:19042
|
|
keyspace: ks1
|
|
"###;
|
|
let res: Result<ChannelConfig, _> = serde_yaml::from_slice(conf.as_bytes());
|
|
assert_eq!(res.is_ok(), true);
|
|
let conf = res.unwrap();
|
|
assert_eq!(conf.api_bind, Some("0.0.0.0:3011".to_string()));
|
|
assert_eq!(conf.search.get(0), Some(&"172.26.0.255".to_string()));
|
|
assert_eq!(conf.scylla.hosts.get(1), Some(&"sf-nube-12:19042".to_string()));
|
|
}
|
|
|
|
pub struct ListenFromFileOpts {
|
|
pub config: PathBuf,
|
|
}
|
|
|
|
fn local_hostname() -> String {
|
|
let mut buf = vec![0u8; 128];
|
|
let hostname = unsafe {
|
|
let ec = libc::gethostname(buf.as_mut_ptr() as _, buf.len() - 2);
|
|
if ec != 0 {
|
|
panic!();
|
|
}
|
|
let hostname = CStr::from_ptr(&buf[0] as *const _ as _);
|
|
hostname.to_str().unwrap()
|
|
};
|
|
hostname.into()
|
|
}
|
|
|
|
#[test]
|
|
fn test_get_local_hostname() {
|
|
assert_ne!(local_hostname().len(), 0);
|
|
}
|
|
|
|
pub async fn parse_config(config: PathBuf) -> Result<CaConnectOpts, Error> {
|
|
let mut file = OpenOptions::new().read(true).open(config).await?;
|
|
let mut buf = vec![];
|
|
file.read_to_end(&mut buf).await?;
|
|
let mut conf: ChannelConfig =
|
|
serde_yaml::from_slice(&buf).map_err(|e| Error::with_msg_no_trace(format!("{:?}", e)))?;
|
|
let re_p = regex::Regex::new(&conf.whitelist.unwrap_or("--nothing-whitelisted--".into()))?;
|
|
let re_n = regex::Regex::new(&conf.blacklist.unwrap_or("--nothing-blacklisted--".into()))?;
|
|
conf.channels = conf
|
|
.channels
|
|
.into_iter()
|
|
.filter(|ch| {
|
|
if let Some(_cs) = re_p.captures(&ch) {
|
|
true
|
|
} else if re_n.is_match(&ch) {
|
|
false
|
|
} else {
|
|
true
|
|
}
|
|
})
|
|
.collect();
|
|
Ok(CaConnectOpts {
|
|
backend: conf.backend,
|
|
channels: conf.channels,
|
|
search: conf.search,
|
|
search_blacklist: conf.search_blacklist,
|
|
addr_bind: conf.addr_bind.unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))),
|
|
addr_conn: conf.addr_conn.unwrap_or(IpAddr::V4(Ipv4Addr::new(255, 255, 255, 255))),
|
|
timeout: conf.timeout.unwrap_or(1200),
|
|
pgconf: conf.postgresql,
|
|
scyconf: conf.scylla,
|
|
array_truncate: conf.array_truncate.unwrap_or(512),
|
|
insert_worker_count: conf.insert_worker_count.unwrap_or(800),
|
|
insert_scylla_sessions: conf.insert_scylla_sessions.unwrap_or(1),
|
|
insert_queue_max: conf.insert_queue_max.unwrap_or(64),
|
|
insert_item_queue_cap: conf.insert_item_queue_cap.unwrap_or(200000),
|
|
api_bind: conf.api_bind.unwrap_or_else(|| "0.0.0.0:3011".into()),
|
|
local_epics_hostname: conf.local_epics_hostname.unwrap_or_else(local_hostname),
|
|
})
|
|
}
|
|
|
|
pub struct CaConnectOpts {
|
|
pub backend: String,
|
|
pub channels: Vec<String>,
|
|
pub search: Vec<String>,
|
|
pub search_blacklist: Vec<String>,
|
|
pub addr_bind: IpAddr,
|
|
pub addr_conn: IpAddr,
|
|
pub timeout: u64,
|
|
pub pgconf: Database,
|
|
pub scyconf: ScyllaConfig,
|
|
pub array_truncate: usize,
|
|
pub insert_worker_count: usize,
|
|
pub insert_scylla_sessions: usize,
|
|
pub insert_queue_max: usize,
|
|
pub insert_item_queue_cap: usize,
|
|
pub api_bind: String,
|
|
pub local_epics_hostname: String,
|
|
}
|
|
|
|
async fn spawn_scylla_insert_workers(
|
|
scyconf: ScyllaConfig,
|
|
insert_scylla_sessions: usize,
|
|
insert_worker_count: usize,
|
|
insert_item_queue: Arc<CommonInsertItemQueue>,
|
|
insert_frac: Arc<AtomicU64>,
|
|
pg_client: Arc<PgClient>,
|
|
store_stats: Arc<stats::CaConnStats>,
|
|
) -> Result<Vec<JoinHandle<()>>, Error> {
|
|
let mut jhs = Vec::new();
|
|
let mut data_stores = Vec::new();
|
|
for _ in 0..insert_scylla_sessions {
|
|
let data_store = Arc::new(DataStore::new(&scyconf, pg_client.clone()).await?);
|
|
data_stores.push(data_store);
|
|
}
|
|
for i1 in 0..insert_worker_count {
|
|
let data_store = data_stores[i1 * data_stores.len() / insert_worker_count].clone();
|
|
let stats = store_stats.clone();
|
|
let recv = insert_item_queue.receiver();
|
|
let insert_frac = insert_frac.clone();
|
|
let fut = async move {
|
|
let mut i1 = 0;
|
|
while let Ok(item) = recv.recv().await {
|
|
match item {
|
|
QueryItem::ConnectionStatus(item) => {
|
|
match crate::store::insert_connection_status(item, &data_store, &stats).await {
|
|
Ok(_) => {
|
|
stats.store_worker_item_insert_inc();
|
|
}
|
|
Err(e) => {
|
|
stats.store_worker_item_error_inc();
|
|
// TODO introduce more structured error variants.
|
|
if e.msg().contains("WriteTimeout") {
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
} else {
|
|
// TODO back off but continue.
|
|
error!("insert worker sees error: {e:?}");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
QueryItem::ChannelStatus(item) => {
|
|
match crate::store::insert_channel_status(item, &data_store, &stats).await {
|
|
Ok(_) => {
|
|
stats.store_worker_item_insert_inc();
|
|
}
|
|
Err(e) => {
|
|
stats.store_worker_item_error_inc();
|
|
// TODO introduce more structured error variants.
|
|
if e.msg().contains("WriteTimeout") {
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
} else {
|
|
// TODO back off but continue.
|
|
error!("insert worker sees error: {e:?}");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
QueryItem::Insert(item) => {
|
|
stats.store_worker_item_recv_inc();
|
|
let insert_frac = insert_frac.load(Ordering::Acquire);
|
|
if i1 % 1000 < insert_frac {
|
|
match crate::store::insert_item(item, &data_store, &stats).await {
|
|
Ok(_) => {
|
|
stats.store_worker_item_insert_inc();
|
|
}
|
|
Err(e) => {
|
|
stats.store_worker_item_error_inc();
|
|
// TODO introduce more structured error variants.
|
|
if e.msg().contains("WriteTimeout") {
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
} else {
|
|
// TODO back off but continue.
|
|
error!("insert worker sees error: {e:?}");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
stats.store_worker_item_drop_inc();
|
|
}
|
|
i1 += 1;
|
|
}
|
|
QueryItem::Mute(item) => {
|
|
let values = (
|
|
(item.series.id() & 0xff) as i32,
|
|
item.series.id() as i64,
|
|
item.ts as i64,
|
|
item.ema,
|
|
item.emd,
|
|
);
|
|
let qres = data_store
|
|
.scy
|
|
.query(
|
|
"insert into muted (part, series, ts, ema, emd) values (?, ?, ?, ?, ?)",
|
|
values,
|
|
)
|
|
.await;
|
|
match qres {
|
|
Ok(_) => {}
|
|
Err(_) => {
|
|
stats.store_worker_item_error_inc();
|
|
}
|
|
}
|
|
}
|
|
QueryItem::Ivl(item) => {
|
|
let values = (
|
|
(item.series.id() & 0xff) as i32,
|
|
item.series.id() as i64,
|
|
item.ts as i64,
|
|
item.ema,
|
|
item.emd,
|
|
);
|
|
let qres = data_store
|
|
.scy
|
|
.query(
|
|
"insert into item_recv_ivl (part, series, ts, ema, emd) values (?, ?, ?, ?, ?)",
|
|
values,
|
|
)
|
|
.await;
|
|
match qres {
|
|
Ok(_) => {}
|
|
Err(_) => {
|
|
stats.store_worker_item_error_inc();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
info!("insert worker has no more messages");
|
|
};
|
|
let jh = tokio::spawn(fut);
|
|
jhs.push(jh);
|
|
}
|
|
Ok(jhs)
|
|
}
|
|
|
|
pub struct CommandQueueSet {
|
|
queues: tokio::sync::Mutex<BTreeMap<SocketAddrV4, Sender<ConnCommand>>>,
|
|
}
|
|
|
|
impl CommandQueueSet {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
queues: tokio::sync::Mutex::new(BTreeMap::<SocketAddrV4, Sender<ConnCommand>>::new()),
|
|
}
|
|
}
|
|
|
|
pub async fn queues(&self) -> &tokio::sync::Mutex<BTreeMap<SocketAddrV4, Sender<ConnCommand>>> {
|
|
&self.queues
|
|
}
|
|
|
|
pub async fn queues_locked(&self) -> tokio::sync::MutexGuard<BTreeMap<SocketAddrV4, Sender<ConnCommand>>> {
|
|
let mut g = self.queues.lock().await;
|
|
let mut rm = Vec::new();
|
|
for (k, v) in g.iter() {
|
|
if v.is_closed() {
|
|
rm.push(*k);
|
|
}
|
|
}
|
|
for x in rm {
|
|
g.remove(&x);
|
|
}
|
|
g
|
|
}
|
|
}
|
|
|
|
pub struct IngestCommons {
|
|
pub pgconf: Arc<Database>,
|
|
pub local_epics_hostname: String,
|
|
pub insert_item_queue: Arc<CommonInsertItemQueue>,
|
|
pub data_store: Arc<DataStore>,
|
|
pub insert_ivl_min: Arc<AtomicU64>,
|
|
pub conn_stats: Arc<TokMx<Vec<Arc<CaConnStats>>>>,
|
|
pub command_queue_set: Arc<CommandQueueSet>,
|
|
}
|
|
|
|
pub async fn find_channel_addr(
|
|
backend: String,
|
|
name: String,
|
|
pgconf: &Database,
|
|
) -> Result<Option<SocketAddrV4>, Error> {
|
|
// TODO also here, provide a db pool.
|
|
let d = pgconf;
|
|
let (pg_client, pg_conn) = tokio_postgres::connect(
|
|
&format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name),
|
|
tokio_postgres::tls::NoTls,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
// TODO allow clean shutdown on ctrl-c and join the pg_conn in the end:
|
|
tokio::spawn(pg_conn);
|
|
let pg_client = Arc::new(pg_client);
|
|
let qu_find_addr = pg_client
|
|
.prepare(
|
|
"select t1.facility, t1.channel, t1.addr from ioc_by_channel t1 where t1.facility = $1 and t1.channel = $2",
|
|
)
|
|
.await
|
|
.err_conv()?;
|
|
let rows = pg_client.query(&qu_find_addr, &[&backend, &name]).await.err_conv()?;
|
|
if rows.is_empty() {
|
|
error!("can not find any addresses of channels {:?}", name);
|
|
Err(Error::with_msg_no_trace(format!("no address for channel {}", name)))
|
|
} else {
|
|
for row in rows {
|
|
let addr: &str = row.get(2);
|
|
if addr == "" {
|
|
return Err(Error::with_msg_no_trace(format!("no address for channel {}", name)));
|
|
} else {
|
|
match addr.parse::<SocketAddrV4>() {
|
|
Ok(addr) => return Ok(Some(addr)),
|
|
Err(e) => {
|
|
error!("can not parse {e:?}");
|
|
return Err(Error::with_msg_no_trace(format!("no address for channel {}", name)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
pub async fn create_ca_conn(
|
|
addr: SocketAddrV4,
|
|
local_epics_hostname: String,
|
|
array_truncate: usize,
|
|
insert_queue_max: usize,
|
|
insert_item_queue: Arc<CommonInsertItemQueue>,
|
|
data_store: Arc<DataStore>,
|
|
insert_ivl_min: Arc<AtomicU64>,
|
|
conn_stats: Arc<TokMx<Vec<Arc<CaConnStats>>>>,
|
|
command_queue_set: Arc<CommandQueueSet>,
|
|
) -> Result<JoinHandle<Result<(), Error>>, Error> {
|
|
info!("create new CaConn {:?}", addr);
|
|
let data_store = data_store.clone();
|
|
let conn = CaConn::new(
|
|
addr,
|
|
local_epics_hostname,
|
|
data_store.clone(),
|
|
insert_item_queue.sender(),
|
|
array_truncate,
|
|
insert_queue_max,
|
|
insert_ivl_min.clone(),
|
|
);
|
|
conn_stats.lock().await.push(conn.stats());
|
|
let stats2 = conn.stats();
|
|
let conn_command_tx = conn.conn_command_tx();
|
|
{
|
|
//command_queue_set.queues().lock().await.insert(addr, conn_command_tx);
|
|
command_queue_set.queues_locked().await.insert(addr, conn_command_tx);
|
|
}
|
|
let conn_block = async move {
|
|
let mut conn = conn;
|
|
while let Some(item) = conn.next().await {
|
|
match item {
|
|
Ok(_) => {
|
|
stats2.conn_item_count_inc();
|
|
}
|
|
Err(e) => {
|
|
error!("CaConn gives error: {e:?}");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
Ok::<_, Error>(())
|
|
};
|
|
let jh = tokio::spawn(conn_block);
|
|
Ok(jh)
|
|
}
|
|
|
|
pub static SIGINT: AtomicU32 = AtomicU32::new(0);
|
|
|
|
fn handler_sigaction(_a: libc::c_int, _b: *const libc::siginfo_t, _c: *const libc::c_void) {
|
|
SIGINT.store(1, Ordering::Release);
|
|
let _ = unset_signal_handler();
|
|
}
|
|
|
|
fn set_signal_handler() -> Result<(), Error> {
|
|
let mask: libc::sigset_t = unsafe { MaybeUninit::zeroed().assume_init() };
|
|
let handler: libc::sighandler_t = handler_sigaction as *const libc::c_void as _;
|
|
let act = libc::sigaction {
|
|
sa_sigaction: handler,
|
|
sa_mask: mask,
|
|
sa_flags: 0,
|
|
sa_restorer: None,
|
|
};
|
|
unsafe {
|
|
let ec = libc::sigaction(libc::SIGINT, &act, std::ptr::null_mut());
|
|
if ec != 0 {
|
|
let errno = *libc::__errno_location();
|
|
let msg = CStr::from_ptr(libc::strerror(errno));
|
|
error!("error: {:?}", msg);
|
|
return Err(Error::with_msg_no_trace(format!("can not set signal handler")));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn unset_signal_handler() -> Result<(), Error> {
|
|
let mask: libc::sigset_t = unsafe { MaybeUninit::zeroed().assume_init() };
|
|
let act = libc::sigaction {
|
|
sa_sigaction: libc::SIG_DFL,
|
|
sa_mask: mask,
|
|
sa_flags: 0,
|
|
sa_restorer: None,
|
|
};
|
|
unsafe {
|
|
let ec = libc::sigaction(libc::SIGINT, &act, std::ptr::null_mut());
|
|
if ec != 0 {
|
|
let errno = *libc::__errno_location();
|
|
let msg = CStr::from_ptr(libc::strerror(errno));
|
|
error!("error: {:?}", msg);
|
|
return Err(Error::with_msg_no_trace(format!("can not set signal handler")));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn ca_connect(opts: ListenFromFileOpts) -> Result<(), Error> {
|
|
set_signal_handler()?;
|
|
let insert_frac = Arc::new(AtomicU64::new(1000));
|
|
let insert_ivl_min = Arc::new(AtomicU64::new(8800));
|
|
let opts = parse_config(opts.config).await?;
|
|
let scyconf = opts.scyconf.clone();
|
|
|
|
let pgconf = Database {
|
|
name: opts.pgconf.name.clone(),
|
|
host: opts.pgconf.host.clone(),
|
|
port: opts.pgconf.port.clone(),
|
|
user: opts.pgconf.user.clone(),
|
|
pass: opts.pgconf.pass.clone(),
|
|
};
|
|
|
|
let d = &pgconf;
|
|
let (pg_client, pg_conn) = tokio_postgres::connect(
|
|
&format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name),
|
|
tokio_postgres::tls::NoTls,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
// TODO allow clean shutdown on ctrl-c and join the pg_conn in the end:
|
|
tokio::spawn(pg_conn);
|
|
let pg_client = Arc::new(pg_client);
|
|
|
|
// TODO use new struct:
|
|
let local_stats = Arc::new(CaConnStats::new());
|
|
// TODO factor the find loop into a separate Stream.
|
|
let qu_find_addr = pg_client
|
|
.prepare("with q1 as (select t1.facility, t1.channel, t1.addr from ioc_by_channel t1 where t1.facility = $1 and t1.channel in ($2, $3, $4, $5, $6, $7, $8, $9) and t1.addr != '' order by t1.tsmod desc) select distinct on (q1.facility, q1.channel) q1.facility, q1.channel, q1.addr from q1")
|
|
.await
|
|
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
|
|
|
|
// Fetch all addresses for all channels.
|
|
let rows = pg_client
|
|
.query("select channel, addr from ioc_by_channel", &[])
|
|
.await
|
|
.err_conv()?;
|
|
let mut phonebook = BTreeMap::new();
|
|
for row in rows {
|
|
let channel: String = row.get(0);
|
|
let addr: String = row.get(1);
|
|
let addr: SocketAddrV4 = addr
|
|
.parse()
|
|
.map_err(|_| Error::with_msg_no_trace(format!("can not parse address {addr}")))?;
|
|
phonebook.insert(channel, addr);
|
|
}
|
|
|
|
let mut channels_by_host = BTreeMap::new();
|
|
|
|
let data_store = Arc::new(DataStore::new(&scyconf, pg_client.clone()).await?);
|
|
let insert_item_queue = CommonInsertItemQueue::new(opts.insert_item_queue_cap);
|
|
let insert_item_queue = Arc::new(insert_item_queue);
|
|
// TODO use a new stats struct
|
|
let store_stats = Arc::new(CaConnStats::new());
|
|
let jh_insert_workers = spawn_scylla_insert_workers(
|
|
opts.scyconf.clone(),
|
|
opts.insert_scylla_sessions,
|
|
opts.insert_worker_count,
|
|
insert_item_queue.clone(),
|
|
insert_frac.clone(),
|
|
pg_client.clone(),
|
|
store_stats.clone(),
|
|
)
|
|
.await?;
|
|
|
|
let mut conn_jhs = vec![];
|
|
let conn_stats: Arc<TokMx<Vec<Arc<CaConnStats>>>> = Arc::new(TokMx::new(Vec::new()));
|
|
let command_queue_set = Arc::new(CommandQueueSet::new());
|
|
|
|
let ingest_commons = IngestCommons {
|
|
pgconf: Arc::new(pgconf.clone()),
|
|
local_epics_hostname: opts.local_epics_hostname.clone(),
|
|
insert_item_queue: insert_item_queue.clone(),
|
|
data_store: data_store.clone(),
|
|
insert_ivl_min: insert_ivl_min.clone(),
|
|
conn_stats: conn_stats.clone(),
|
|
command_queue_set: command_queue_set.clone(),
|
|
};
|
|
let ingest_commons = Arc::new(ingest_commons);
|
|
|
|
if true {
|
|
tokio::spawn(crate::metrics::start_metrics_service(
|
|
opts.api_bind.clone(),
|
|
insert_frac.clone(),
|
|
insert_ivl_min.clone(),
|
|
ingest_commons.clone(),
|
|
));
|
|
}
|
|
|
|
let metrics_agg_fut = {
|
|
let conn_stats = conn_stats.clone();
|
|
let local_stats = local_stats.clone();
|
|
async move {
|
|
let mut agg_last = CaConnStatsAgg::new();
|
|
loop {
|
|
tokio::time::sleep(Duration::from_millis(671)).await;
|
|
let agg = CaConnStatsAgg::new();
|
|
agg.push(&local_stats);
|
|
agg.push(&store_stats);
|
|
for g in conn_stats.lock().await.iter() {
|
|
agg.push(&g);
|
|
}
|
|
let m = get_metrics();
|
|
*m = Some(agg.clone());
|
|
if false {
|
|
let diff = CaConnStatsAggDiff::diff_from(&agg_last, &agg);
|
|
info!("{}", diff.display());
|
|
}
|
|
agg_last = agg;
|
|
if false {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
};
|
|
let metrics_agg_jh = tokio::spawn(metrics_agg_fut);
|
|
|
|
let mut chns_todo = &opts.channels[..];
|
|
let mut chstmp = ["__NONE__"; 8];
|
|
let mut ix = 0;
|
|
while chns_todo.len() > 0 && SIGINT.load(Ordering::Acquire) == 0 {
|
|
if false {
|
|
for (s1, s2) in chns_todo.iter().zip(chstmp.iter_mut()) {
|
|
*s2 = s1;
|
|
}
|
|
chns_todo = &chns_todo[chstmp.len().min(chns_todo.len())..];
|
|
let rows = pg_client
|
|
.query(
|
|
&qu_find_addr,
|
|
&[
|
|
&opts.backend,
|
|
&chstmp[0],
|
|
&chstmp[1],
|
|
&chstmp[2],
|
|
&chstmp[3],
|
|
&chstmp[4],
|
|
&chstmp[5],
|
|
&chstmp[6],
|
|
&chstmp[7],
|
|
],
|
|
)
|
|
.await
|
|
.map_err(|e| Error::with_msg_no_trace(format!("pg lookup error: {e:?}")))?;
|
|
for row in rows {
|
|
let ch: &str = row.get(1);
|
|
let addr: &str = row.get(2);
|
|
if addr == "" {
|
|
// TODO the address was searched before but could not be found.
|
|
} else {
|
|
let addr: SocketAddrV4 = match addr.parse() {
|
|
Ok(k) => k,
|
|
Err(e) => {
|
|
error!("can not parse {addr:?} for channel {ch:?} {e:?}");
|
|
continue;
|
|
}
|
|
};
|
|
let _ = addr;
|
|
}
|
|
}
|
|
}
|
|
if let Some(ch) = chns_todo.first() {
|
|
let ch = ch.clone();
|
|
chns_todo = &chns_todo[1..];
|
|
if let Some(addr) = phonebook.get(&ch) {
|
|
if !channels_by_host.contains_key(&addr) {
|
|
channels_by_host.insert(addr, vec![ch.to_string()]);
|
|
} else {
|
|
channels_by_host.get_mut(&addr).unwrap().push(ch.to_string());
|
|
}
|
|
let create_new = {
|
|
let g = command_queue_set.queues_locked().await;
|
|
if let Some(tx) = g.get(&addr) {
|
|
let (cmd, rx) = ConnCommand::channel_add(ch.to_string());
|
|
tx.send(cmd).await.unwrap();
|
|
if !rx.recv().await.unwrap() {
|
|
error!("Could not add channel: {}", ch);
|
|
}
|
|
false
|
|
} else {
|
|
true
|
|
}
|
|
};
|
|
if create_new {
|
|
info!("create new CaConn {:?} {:?}", addr, ch);
|
|
let data_store = data_store.clone();
|
|
let conn = CaConn::new(
|
|
addr.clone(),
|
|
opts.local_epics_hostname.clone(),
|
|
data_store.clone(),
|
|
insert_item_queue.sender(),
|
|
opts.array_truncate,
|
|
opts.insert_queue_max,
|
|
insert_ivl_min.clone(),
|
|
);
|
|
conn_stats.lock().await.push(conn.stats());
|
|
let stats2 = conn.stats();
|
|
let conn_command_tx = conn.conn_command_tx();
|
|
let tx = conn_command_tx.clone();
|
|
{
|
|
command_queue_set
|
|
.queues_locked()
|
|
.await
|
|
.insert(addr.clone(), conn_command_tx);
|
|
}
|
|
let conn_block = async move {
|
|
let mut conn = conn;
|
|
while let Some(item) = conn.next().await {
|
|
match item {
|
|
Ok(_) => {
|
|
stats2.conn_item_count_inc();
|
|
}
|
|
Err(e) => {
|
|
error!("CaConn gives error: {e:?}");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
Ok::<_, Error>(())
|
|
};
|
|
let jh = tokio::spawn(conn_block);
|
|
conn_jhs.push(jh);
|
|
{
|
|
let (cmd, rx) = ConnCommand::channel_add(ch.to_string());
|
|
tx.send(cmd).await.unwrap();
|
|
if !rx.recv().await.unwrap() {
|
|
error!("Could not add channel: {}", ch);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
ix += 1;
|
|
if ix % 1000 == 0 {
|
|
info!("{} of {} {}", ix, opts.channels.len(), ch);
|
|
}
|
|
}
|
|
}
|
|
info!("channels_by_host len {}", channels_by_host.len());
|
|
|
|
let mut conn_jhs: VecDeque<Fuse<JoinHandle<Result<(), Error>>>> =
|
|
conn_jhs.into_iter().map(|jh| jh.fuse()).collect();
|
|
let mut sent_stop_commands = false;
|
|
loop {
|
|
if SIGINT.load(Ordering::Acquire) != 0 {
|
|
if false {
|
|
let receiver = insert_item_queue.receiver();
|
|
let sc = receiver.sender_count();
|
|
let rc = receiver.receiver_count();
|
|
info!("item queue senders {} receivers {}", sc, rc);
|
|
}
|
|
if !sent_stop_commands {
|
|
sent_stop_commands = true;
|
|
info!("sending stop command");
|
|
let queues = command_queue_set.queues_locked().await;
|
|
for q in queues.iter() {
|
|
let (cmd, _rx) = ConnCommand::shutdown();
|
|
let _ = q.1.send(cmd).await;
|
|
}
|
|
}
|
|
}
|
|
let mut jh = if let Some(x) = conn_jhs.pop_front() {
|
|
x
|
|
} else {
|
|
break;
|
|
};
|
|
futures_util::select! {
|
|
a = jh => match a {
|
|
Ok(k) => match k {
|
|
Ok(_) => {}
|
|
Err(e) => {
|
|
error!("{e:?}");
|
|
}
|
|
},
|
|
Err(e) => {
|
|
error!("{e:?}");
|
|
}
|
|
},
|
|
_b = tokio::time::sleep(Duration::from_millis(1000)).fuse() => {
|
|
conn_jhs.push_back(jh);
|
|
if sent_stop_commands {
|
|
info!("waiting for {} connections", conn_jhs.len());
|
|
}
|
|
}
|
|
};
|
|
}
|
|
info!("all connections done.");
|
|
|
|
drop(ingest_commons);
|
|
metrics_agg_jh.abort();
|
|
drop(metrics_agg_jh);
|
|
|
|
if false {
|
|
let sender = insert_item_queue.sender_raw();
|
|
sender.close();
|
|
let receiver = insert_item_queue.receiver();
|
|
receiver.close();
|
|
}
|
|
if true {
|
|
let receiver = insert_item_queue.receiver();
|
|
let sc = receiver.sender_count();
|
|
let rc = receiver.receiver_count();
|
|
info!("item queue A senders {} receivers {}", sc, rc);
|
|
}
|
|
let receiver = insert_item_queue.receiver();
|
|
drop(insert_item_queue);
|
|
if true {
|
|
let sc = receiver.sender_count();
|
|
let rc = receiver.receiver_count();
|
|
info!("item queue B senders {} receivers {}", sc, rc);
|
|
}
|
|
|
|
let mut futs = FuturesUnordered::from_iter(jh_insert_workers);
|
|
loop {
|
|
futures_util::select!(
|
|
x = futs.next() => match x {
|
|
Some(Ok(_)) => {}
|
|
Some(Err(e)) => {
|
|
error!("error on shutdown: {e:?}");
|
|
}
|
|
None => break,
|
|
},
|
|
_ = tokio::time::sleep(Duration::from_millis(1000)).fuse() => {
|
|
info!("waiting for {} inserters", futs.len());
|
|
if true {
|
|
let sc = receiver.sender_count();
|
|
let rc = receiver.receiver_count();
|
|
info!("item queue B senders {} receivers {}", sc, rc);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
info!("all insert workers done.");
|
|
Ok(())
|
|
}
|