daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Update deps, status and hashmap

Browse Source
This commit is contained in:
Dominik Werder
2024-02-07 11:48:40 +01:00
parent f5ca38caaf
commit 3c4970aa2c
9 changed files with 956 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1136
.cargo/cargo-lock
View File

File diff suppressed because it is too large Load Diff

12
dbpg/src/seriesbychannel.rs
View File

@@ -116,7 +116,7 @@ enum MatchingSeries {
Latest(SeriesId),
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub enum RegisteredSeries {
Created(SeriesId),
Updated(SeriesId),
@@ -745,9 +745,19 @@ fn test_series_by_channel_01() {
};
rxs.push(rx);
let mut series_ids = Vec::new();
for rx in rxs {
let res = rx.recv().await.unwrap();
debug!("received A: {res:?}");
series_ids.push(res.unwrap().series);
}
{
let exp = vec![
RegisteredSeries::Created(SeriesId::new(5191776216635917420)),
RegisteredSeries::Created(SeriesId::new(5025677968516078602)),
RegisteredSeries::Updated(SeriesId::new(4802468414253815536)),
];
assert_eq!(series_ids, exp);
}
let (tx, rx) = async_channel::bounded(1);

1
netfetch/Cargo.toml
View File

@@ -33,6 +33,7 @@ lazy_static = "1"
libc = "0.2"
slidebuf = "0.0.1"
dashmap = "5.5.3"
hashbrown = "0.14.3"
log = { path = "../log" }
series = { path = "../series" }
serieswriter = { path = "../serieswriter" }

6
netfetch/src/ca/conn.rs
View File

@@ -15,6 +15,8 @@ use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use hashbrown::HashMap;
use hashbrown::HashSet;
use log::*;
use netpod::timeunits::*;
use netpod::ScalarType;
@@ -53,8 +55,6 @@ use stats::CaConnStats;
use stats::CaProtoStats;
use stats::IntervalEma;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::net::SocketAddrV4;
use std::ops::ControlFlow;
@@ -1531,7 +1531,7 @@ impl CaConn {
let sid_ev = Sid(ev.sid);
let ioid = Ioid(ev.ioid);
if let Some(cid) = self.read_ioids.get(&ioid) {
let ch_s = if let Some(x) = self.channels.get_mut(&cid) {
let ch_s = if let Some(x) = self.channels.get_mut(cid) {
x
} else {
warn!("handle_read_notify_res can not find channel for {cid:?} {ioid:?}");

6
netfetch/src/ca/connset.rs
View File

@@ -30,6 +30,7 @@ use err::Error;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
use hashbrown::HashMap;
use log::*;
use scywr::iteminsertqueue::ChannelInfoItem;
use scywr::iteminsertqueue::ChannelStatusItem;
@@ -54,7 +55,6 @@ use stats::CaConnStats;
use stats::CaProtoStats;
use stats::IocFinderStats;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::net::SocketAddr;
use std::net::SocketAddrV4;
@@ -346,7 +346,7 @@ pub struct CaConnSet {
ticker: Pin<Box<tokio::time::Sleep>>,
backend: String,
local_epics_hostname: String,
ca_conn_ress: BTreeMap<SocketAddr, CaConnRes>,
ca_conn_ress: HashMap<SocketAddr, CaConnRes>,
channel_states: ChannelStateMap,
channel_by_cssid: HashMap<ChannelStatusSeriesId, Channel>,
connset_inp_rx: Pin<Box<Receiver<CaConnSetEvent>>>,
@@ -413,7 +413,7 @@ impl CaConnSet {
ticker: Self::new_self_ticker(),
backend,
local_epics_hostname,
ca_conn_ress: BTreeMap::new(),
ca_conn_ress: HashMap::new(),
channel_states: ChannelStateMap::new(),
channel_by_cssid: HashMap::new(),
connset_inp_rx: Box::pin(connset_inp_rx),

24
netfetch/src/ca/finder.rs
View File

@@ -11,10 +11,10 @@ use dbpg::iocindex::IocItem;
use dbpg::iocindex::IocSearchIndexWorker;
use dbpg::postgres::Row as PgRow;
use err::Error;
use hashbrown::HashMap;
use log::*;
use netpod::Database;
use stats::IocFinderStats;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::Duration;
@@ -188,17 +188,19 @@ async fn finder_worker_single(
let nbatch = batch.len();
trace_batch!("received results {} resdiff {}", rows.len(), resdiff);
let items = transform_pgres(rows);
let names: HashMap<_, _> = items.iter().map(|x| (&x.channel, true)).collect();
let mut to_add = Vec::new();
for e in batch {
if !names.contains_key(e.name_string()) {
let item = FindIocRes {
channel: e.name().into(),
response_addr: None,
addr: None,
dt: Duration::from_millis(0),
};
to_add.push(item);
{
let names: HashMap<_, _> = items.iter().map(|x| (&x.channel, true)).collect();
for e in batch {
if !names.contains_key(e.name_string()) {
let item = FindIocRes {
channel: e.name().into(),
response_addr: None,
addr: None,
dt: Duration::from_millis(0),
};
to_add.push(item);
}
}
}
let mut items = items;

8
netfetch/src/ca/statemap.rs
View File

@@ -67,10 +67,6 @@ pub struct UnassignedState {
#[derive(Debug, Clone, Serialize)]
pub enum WithStatusSeriesIdStateInner {
UnknownAddress {
#[serde(with = "humantime_serde")]
since: SystemTime,
},
AddrSearchPending {
#[serde(with = "humantime_serde")]
since: SystemTime,
@@ -79,6 +75,10 @@ pub enum WithStatusSeriesIdStateInner {
addr: SocketAddrV4,
state: WithAddressState,
},
UnknownAddress {
#[serde(with = "humantime_serde")]
since: SystemTime,
},
NoAddress {
#[serde(with = "humantime_serde")]
since: SystemTime,

13
netfetch/src/metrics.rs
View File

@@ -1,3 +1,5 @@
pub mod status;
use crate::ca::conn::ChannelStateInfo;
use crate::ca::connset::CaConnSetEvent;
use crate::ca::connset::ChannelStatusesRequest;
@@ -187,7 +189,7 @@ async fn channel_remove(params: HashMap<String, String>, dcom: Arc<DaemonComm>)
// ChannelStatusesResponse
// BTreeMap<String, ChannelState>
async fn channel_states(
async fn private_channel_states(
params: HashMap<String, String>,
tx: Sender<CaConnSetEvent>,
) -> axum::Json<BTreeMap<String, ChannelState>> {
@@ -316,7 +318,14 @@ fn make_routes(dcom: Arc<DaemonComm>, connset_cmd_tx: Sender<CaConnSetEvent>, st
"/daqingest/channel/states",
get({
let tx = connset_cmd_tx.clone();
|Query(params): Query<HashMap<String, String>>| channel_states(params, tx)
|Query(params): Query<HashMap<String, String>>| status::channel_states(params, tx)
}),
)
.route(
"/daqingest/private/channel/states",
get({
let tx = connset_cmd_tx.clone();
|Query(params): Query<HashMap<String, String>>| private_channel_states(params, tx)
}),
)
.route(

194
netfetch/src/metrics/status.rs Normal file
View File

@@ -0,0 +1,194 @@
use crate::ca::connset::CaConnSetEvent;
use crate::ca::connset::ChannelStatusesRequest;
use crate::ca::connset::ConnSetCmd;
use async_channel::Sender;
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::net::SocketAddr;
#[derive(Debug, Serialize)]
pub struct ChannelStates {
channels: BTreeMap<String, ChannelState>,
}
#[derive(Debug, Serialize)]
struct ChannelState {
ioc_address: Option<SocketAddr>,
connection: ConnectionState,
archive_settings: ArchiveSettings,
}
#[derive(Debug, Serialize)]
enum ConnectionState {
Connecting,
Unreachable,
Disconnected,
Connected,
Error,
}
#[derive(Debug, Serialize)]
struct ArchiveSettings {
short_term: Option<String>,
medium_term: Option<String>,
long_term: Option<String>,
}
impl ArchiveSettings {
fn dummy() -> Self {
Self {
short_term: None,
medium_term: None,
long_term: None,
}
}
}
// ChannelStatusesResponse
// BTreeMap<String, ChannelState>
pub async fn channel_states(params: HashMap<String, String>, tx: Sender<CaConnSetEvent>) -> axum::Json<ChannelStates> {
let name = params.get("name").map_or(String::new(), |x| x.clone()).to_string();
let limit = params
.get("limit")
.map(|x| x.parse().ok())
.unwrap_or(None)
.unwrap_or(40);
let (tx2, rx2) = async_channel::bounded(1);
let req = ChannelStatusesRequest { name, limit, tx: tx2 };
let item = CaConnSetEvent::ConnSetCmd(ConnSetCmd::ChannelStatuses(req));
// TODO handle error
tx.send(item).await.unwrap();
let res = rx2.recv().await.unwrap();
let mut states = ChannelStates {
channels: BTreeMap::new(),
};
for (k, st1) in res.channels_ca_conn_set {
use crate::ca::statemap::ChannelStateValue;
match st1.value {
ChannelStateValue::Active(st2) => {
use crate::ca::statemap::ActiveChannelState;
match st2 {
ActiveChannelState::Init { .. } => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ActiveChannelState::WaitForStatusSeriesId { .. } => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ActiveChannelState::WithStatusSeriesId(st3) => {
use crate::ca::statemap::WithStatusSeriesIdStateInner;
match st3.inner {
WithStatusSeriesIdStateInner::AddrSearchPending { .. } => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
WithStatusSeriesIdStateInner::WithAddress { addr, state: st4 } => {
use crate::ca::statemap::WithAddressState;
match st4 {
WithAddressState::Unassigned { .. } => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
WithAddressState::Assigned(st5) => {
use crate::ca::statemap::ConnectionStateValue;
match st5.value {
ConnectionStateValue::Unknown => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ConnectionStateValue::ChannelStateInfo(st6) => {
use crate::ca::conn::ChannelConnectedInfo;
match st6.channel_connected_info {
ChannelConnectedInfo::Disconnected => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Disconnected,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ChannelConnectedInfo::Connecting => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ChannelConnectedInfo::Connected => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Connected,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
ChannelConnectedInfo::Error => {
let chst = ChannelState {
ioc_address: Some(SocketAddr::V4(addr)),
connection: ConnectionState::Error,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
}
}
}
}
}
}
WithStatusSeriesIdStateInner::UnknownAddress { .. } => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Connecting,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
WithStatusSeriesIdStateInner::NoAddress { .. } => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Unreachable,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
WithStatusSeriesIdStateInner::MaybeWrongAddress(..) => {
let chst = ChannelState {
ioc_address: None,
connection: ConnectionState::Unreachable,
archive_settings: ArchiveSettings::dummy(),
};
states.channels.insert(k, chst);
}
}
}
}
}
ChannelStateValue::ToRemove { .. } => {}
}
}
axum::Json(states)
}