daqbuf/daqingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Add more supported data types and stats counter

Browse Source
This commit is contained in:
Dominik Werder
2022-05-24 14:05:36 +02:00
parent 2f9a4092c8
commit 277597400e
9 changed files with 338 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
netfetch/src/ca/conn.rs
+37 -25
View File
@@ -11,7 +11,7 @@ use libc::c_int;
use log::*;
use netpod::timeunits::SEC;
use netpod::{ScalarType, Shape};
use stats::{CaConnStats2, IntervalEma};
use stats::{CaConnStats, IntervalEma};
use std::collections::{BTreeMap, VecDeque};
use std::net::{Ipv4Addr, SocketAddrV4};
use std::pin::Pin;
@@ -22,8 +22,8 @@ use std::time::{Duration, Instant, SystemTime};
use tokio::io::unix::AsyncFd;
use tokio::net::TcpStream;
const INSERT_FUTS_MAX: usize = 200;
const INSERT_FUTS_LIM: usize = 80000;
const INSERT_FUTS_MAX: usize = 2;
const INSERT_FUTS_LIM: usize = 16;
const TABLE_SERIES_MOD: u32 = 128;
#[derive(Debug)]
@@ -105,7 +105,7 @@ macro_rules! insert_scalar_impl {
ts_msp_changed: bool,
st: ScalarType,
sh: Shape,
stats: Arc<CaConnStats2>,
stats: Arc<CaConnStats>,
) {
if futs_queue.len() >= INSERT_FUTS_LIM {
stats.inserts_discard.fetch_add(1, Ordering::AcqRel);
@@ -144,6 +144,7 @@ macro_rules! insert_scalar_impl {
Box::pin(fut3) as _
};
futs_queue.push(fut);
stats.inserts_queue_push_inc();
}
};
}
@@ -162,7 +163,7 @@ macro_rules! insert_array_impl {
ts_msp_changed: bool,
st: ScalarType,
sh: Shape,
stats: Arc<CaConnStats2>,
stats: Arc<CaConnStats>,
) {
if futs_queue.len() >= INSERT_FUTS_LIM {
stats.inserts_discard.fetch_add(1, Ordering::AcqRel);
@@ -201,6 +202,7 @@ macro_rules! insert_array_impl {
Box::pin(fut3) as _
};
futs_queue.push(fut);
stats.inserts_queue_push_inc();
}
};
}
@@ -213,6 +215,8 @@ insert_scalar_impl!(insert_scalar_f64, f64, qu_insert_scalar_f64);
insert_scalar_impl!(insert_scalar_string, String, qu_insert_scalar_string);
insert_array_impl!(insert_array_i8, i8, qu_insert_array_i8);
insert_array_impl!(insert_array_i16, i16, qu_insert_array_i16);
insert_array_impl!(insert_array_i32, i32, qu_insert_array_i32);
insert_array_impl!(insert_array_f32, f32, qu_insert_array_f32);
insert_array_impl!(insert_array_f64, f64, qu_insert_array_f64);
@@ -298,14 +302,19 @@ pub struct CaConn {
FuturesOrdered<Pin<Box<dyn Future<Output = Result<(u32, u32, u16, u16, Existence<SeriesId>), Error>> + Send>>>,
value_insert_futs: FuturesOrdered<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
remote_addr_dbg: SocketAddrV4,
stats2: Arc<CaConnStats2>,
stats: Arc<CaConnStats>,
}
impl CaConn {
pub fn new(tcp: TcpStream, remote_addr_dbg: SocketAddrV4, data_store: Arc<DataStore>) -> Self {
pub fn new(
tcp: TcpStream,
remote_addr_dbg: SocketAddrV4,
data_store: Arc<DataStore>,
array_truncate: usize,
) -> Self {
Self {
state: CaConnState::Init,
proto: CaProto::new(tcp, remote_addr_dbg),
proto: CaProto::new(tcp, remote_addr_dbg, array_truncate),
cid_store: IdStore::new(),
ioid_store: IdStore::new(),
subid_store: IdStore::new(),
@@ -320,12 +329,12 @@ impl CaConn {
fut_get_series: FuturesOrdered::new(),
value_insert_futs: FuturesOrdered::new(),
remote_addr_dbg,
stats2: Arc::new(CaConnStats2::new()),
stats: Arc::new(CaConnStats::new()),
}
}
pub fn stats2(&self) -> Arc<CaConnStats2> {
self.stats2.clone()
pub fn stats(&self) -> Arc<CaConnStats> {
self.stats.clone()
}
pub fn channel_add(&mut self, channel: String) {
@@ -359,7 +368,9 @@ impl CaConn {
while self.value_insert_futs.len() > 0 {
match self.value_insert_futs.poll_next_unpin(cx) {
Pending => break,
_ => {}
_ => {
self.stats.inserts_queue_pop_inc();
}
}
}
Ok(())
@@ -490,7 +501,7 @@ impl CaConn {
ts_msp_changed,
scalar_type,
shape,
self.stats2.clone(),
self.stats.clone(),
);
match ev.value {
Scalar(v) => match v {
@@ -508,6 +519,8 @@ impl CaConn {
use crate::ca::proto::CaDataArrayValue::*;
match v {
I8(val) => match_array_value_insert!(I8, insert_array_i8, val, comm),
I16(val) => match_array_value_insert!(I16, insert_array_i16, val, comm),
I32(val) => match_array_value_insert!(I32, insert_array_i32, val, comm),
F32(val) => match_array_value_insert!(F32, insert_array_f32, val, comm),
F64(val) => match_array_value_insert!(F64, insert_array_f64, val, comm),
_ => {
@@ -662,7 +675,7 @@ impl CaConn {
let mut msgs_tmp = vec![];
self.check_channels_state_init(&mut msgs_tmp)?;
let ts2 = Instant::now();
self.stats2
self.stats
.time_check_channels_state_init
.fetch_add((ts2.duration_since(ts1) * 1000000).as_secs(), Ordering::Release);
ts1 = ts2;
@@ -730,7 +743,7 @@ impl CaConn {
CaMsgTy::EventAddRes(k) => {
let res = Self::handle_event_add_res(self, k);
let ts2 = Instant::now();
self.stats2
self.stats
.time_handle_event_add_res
.fetch_add((ts2.duration_since(ts1) * 1000000).as_secs(), Ordering::Release);
ts1 = ts2;
@@ -774,13 +787,13 @@ impl Stream for CaConn {
let ret = loop {
self.handle_insert_futs(cx)?;
let ts2 = Instant::now();
self.stats2
self.stats
.poll_time_handle_insert_futs
.fetch_add((ts2.duration_since(ts1) * 1000000).as_secs(), Ordering::AcqRel);
ts1 = ts2;
self.handle_get_series_futs(cx)?;
let ts2 = Instant::now();
self.stats2
self.stats
.poll_time_get_series_futs
.fetch_add((ts2.duration_since(ts1) * 1000000).as_secs(), Ordering::AcqRel);
ts1 = ts2;
@@ -805,7 +818,7 @@ impl Stream for CaConn {
CaConnState::Listen => match {
let res = self.handle_conn_listen(cx);
let ts2 = Instant::now();
self.stats2
self.stats
.time_handle_conn_listen
.fetch_add((ts2.duration_since(ts1) * 1000000).as_secs(), Ordering::AcqRel);
ts1 = ts2;
@@ -817,20 +830,19 @@ impl Stream for CaConn {
CaConnState::PeerReady => {
let res = self.handle_peer_ready(cx);
let ts2 = Instant::now();
self.stats2.time_handle_peer_ready_dur(ts2.duration_since(ts1));
self.stats.time_handle_peer_ready_dur(ts2.duration_since(ts1));
ts1 = ts2;
res
}
CaConnState::Done => Ready(None),
};
};
let nn = self.value_insert_futs.len() as u64;
if nn > 1000 {
warn!("insert_queue_len {nn}");
let nn = self.value_insert_futs.len();
if nn > INSERT_FUTS_LIM {
warn!("value_insert_futs len {nn}");
}
self.stats2.inserts_queue_len.store(nn, Ordering::Release);
let ts_outer_2 = Instant::now();
self.stats2.poll_time_all_dur(ts_outer_2.duration_since(ts_outer_1));
self.stats.poll_time_all_dur(ts_outer_2.duration_since(ts_outer_1));
ret
}
}
@@ -1099,7 +1111,7 @@ impl FindIocStream {
error!("incomplete message, missing payload");
break;
}
let msg = CaMsg::from_proto_infos(&hi, nb.data())?;
let msg = CaMsg::from_proto_infos(&hi, nb.data(), 32)?;
nb.adv(hi.payload())?;
msgs.push(msg);
}
netfetch/src/ca/proto.rs
+113 -26
View File
@@ -2,7 +2,7 @@ use crate::netbuf::NetBuf;
use err::Error;
use futures_util::{pin_mut, Stream};
use log::*;
use std::collections::VecDeque;
use std::collections::{BTreeMap, VecDeque};
use std::net::SocketAddrV4;
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -44,6 +44,11 @@ pub struct CreateChanRes {
pub sid: u32,
}
#[derive(Debug)]
pub struct CreateChanFail {
pub cid: u32,
}
#[derive(Debug)]
pub struct AccessRightsRes {
pub cid: u32,
@@ -108,6 +113,8 @@ pub enum CaDataScalarValue {
#[derive(Clone, Debug)]
pub enum CaDataArrayValue {
I8(Vec<i8>),
I16(Vec<i16>),
I32(Vec<i32>),
F32(Vec<f32>),
F64(Vec<f64>),
}
@@ -146,6 +153,7 @@ pub enum CaMsgTy {
SearchRes(SearchRes),
CreateChan(CreateChan),
CreateChanRes(CreateChanRes),
CreateChanFail(CreateChanFail),
AccessRightsRes(AccessRightsRes),
EventAdd(EventAdd),
EventAddRes(EventAddRes),
@@ -166,6 +174,7 @@ impl CaMsgTy {
SearchRes(_) => 0x06,
CreateChan(_) => 0x12,
CreateChanRes(_) => 0x12,
CreateChanFail(_) => 0x1a,
AccessRightsRes(_) => 0x16,
EventAdd(_) => 0x01,
EventAddRes(_) => 0x01,
@@ -190,6 +199,7 @@ impl CaMsgTy {
SearchRes(_) => 8,
CreateChan(x) => (x.channel.len() + 1 + 7) / 8 * 8,
CreateChanRes(_) => 0,
CreateChanFail(_) => 0,
AccessRightsRes(_) => 0,
EventAdd(_) => 16,
EventAddRes(_) => {
@@ -219,6 +229,7 @@ impl CaMsgTy {
SearchRes(x) => x.tcp_port,
CreateChan(_) => 0,
CreateChanRes(x) => x.data_type,
CreateChanFail(_) => 0,
AccessRightsRes(_) => 0,
EventAdd(x) => x.data_type,
EventAddRes(x) => x.data_type,
@@ -239,6 +250,7 @@ impl CaMsgTy {
SearchRes(_) => 0,
CreateChan(_) => 0,
CreateChanRes(x) => x.data_count,
CreateChanFail(_) => 0,
AccessRightsRes(_) => 0,
EventAdd(x) => x.data_count,
EventAddRes(x) => x.data_count,
@@ -259,6 +271,7 @@ impl CaMsgTy {
SearchRes(x) => x.addr,
CreateChan(x) => x.cid,
CreateChanRes(x) => x.cid,
CreateChanFail(x) => x.cid,
AccessRightsRes(x) => x.cid,
EventAdd(x) => x.sid,
EventAddRes(x) => x.status,
@@ -279,6 +292,7 @@ impl CaMsgTy {
SearchRes(x) => x.id,
CreateChan(_) => CA_PROTO_VERSION as _,
CreateChanRes(x) => x.sid,
CreateChanFail(_) => 0,
AccessRightsRes(x) => x.rights,
EventAdd(x) => x.subid,
EventAddRes(x) => x.subid,
@@ -337,10 +351,11 @@ impl CaMsgTy {
unsafe { std::ptr::copy(&d[0] as _, &mut buf[0] as _, d.len()) };
}
CreateChanRes(_) => {}
CreateChanFail(_) => {}
AccessRightsRes(_) => {}
EventAdd(_) => {
// TODO allow to customize the mask. Test if it works.
buf.copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0]);
buf.copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0e, 0, 0]);
}
EventAddRes(_) => {}
ReadNotify(_) => {}
@@ -391,7 +406,7 @@ impl CaMsg {
}
}
pub fn from_proto_infos(hi: &HeadInfo, payload: &[u8]) -> Result<Self, Error> {
pub fn from_proto_infos(hi: &HeadInfo, payload: &[u8], array_truncate: usize) -> Result<Self, Error> {
let msg = match hi.cmdid {
0 => CaMsg {
ty: CaMsgTy::VersionRes(hi.data_count),
@@ -446,12 +461,42 @@ impl CaMsg {
}),
}
}
26 => {
CaMsg {
// TODO use different structs for request and response:
ty: CaMsgTy::CreateChanFail(CreateChanFail { cid: hi.param1 }),
}
}
1 => {
use netpod::Shape;
let ca_st = CaScalarType::from_ca_u16(hi.data_type)?;
let ca_sh = Shape::from_ca_count(hi.data_count)?;
let value = match ca_sh {
Shape::Scalar => match ca_st {
CaScalarType::I8 => {
type ST = i8;
const STL: usize = std::mem::size_of::<ST>();
if payload.len() < STL {
return Err(Error::with_msg_no_trace(format!(
"not enough payload for i8 {}",
payload.len()
)));
}
let v = ST::from_be_bytes(payload[..STL].try_into()?);
CaDataValue::Scalar(CaDataScalarValue::I8(v))
}
CaScalarType::I16 => {
type ST = i16;
const STL: usize = std::mem::size_of::<ST>();
if payload.len() < STL {
return Err(Error::with_msg_no_trace(format!(
"not enough payload for i16 {}",
payload.len()
)));
}
let v = ST::from_be_bytes(payload[..STL].try_into()?);
CaDataValue::Scalar(CaDataScalarValue::I16(v))
}
CaScalarType::I32 => {
type ST = i32;
const STL: usize = std::mem::size_of::<ST>();
@@ -513,18 +558,12 @@ impl CaMsg {
let v = String::from_utf8_lossy(&payload[..ixn]);
CaDataValue::Scalar(CaDataScalarValue::String(v.into()))
}
_ => {
warn!("TODO handle {ca_st:?}");
return Err(Error::with_msg_no_trace(format!(
"can not yet handle type scalar {ca_st:?}"
)));
}
},
Shape::Wave(n) => match ca_st {
CaScalarType::I8 => {
type ST = i8;
const STL: usize = std::mem::size_of::<ST>();
let nn = (n as usize).min(payload.len() / STL);
let nn = (n as usize).min(payload.len() / STL).min(array_truncate);
let mut a = Vec::with_capacity(nn);
let mut bb = &payload[..];
for _ in 0..nn {
@@ -534,10 +573,36 @@ impl CaMsg {
}
CaDataValue::Array(CaDataArrayValue::I8(a))
}
CaScalarType::I16 => {
type ST = i16;
const STL: usize = std::mem::size_of::<ST>();
let nn = (n as usize).min(payload.len() / STL).min(array_truncate);
let mut a = Vec::with_capacity(nn);
let mut bb = &payload[..];
for _ in 0..nn {
let v = ST::from_be_bytes(bb[..STL].try_into()?);
bb = &bb[STL..];
a.push(v);
}
CaDataValue::Array(CaDataArrayValue::I16(a))
}
CaScalarType::I32 => {
type ST = i32;
const STL: usize = std::mem::size_of::<ST>();
let nn = (n as usize).min(payload.len() / STL).min(array_truncate);
let mut a = Vec::with_capacity(nn);
let mut bb = &payload[..];
for _ in 0..nn {
let v = ST::from_be_bytes(bb[..STL].try_into()?);
bb = &bb[STL..];
a.push(v);
}
CaDataValue::Array(CaDataArrayValue::I32(a))
}
CaScalarType::F32 => {
type ST = f32;
const STL: usize = std::mem::size_of::<ST>();
let nn = (n as usize).min(payload.len() / STL);
let nn = (n as usize).min(payload.len() / STL).min(array_truncate);
let mut a = Vec::with_capacity(nn);
let mut bb = &payload[..];
for _ in 0..nn {
@@ -550,7 +615,7 @@ impl CaMsg {
CaScalarType::F64 => {
type ST = f64;
const STL: usize = std::mem::size_of::<ST>();
let nn = (n as usize).min(payload.len() / STL);
let nn = (n as usize).min(payload.len() / STL).min(array_truncate);
let mut a = Vec::with_capacity(nn);
let mut bb = &payload[..];
for _ in 0..nn {
@@ -561,7 +626,7 @@ impl CaMsg {
CaDataValue::Array(CaDataArrayValue::F64(a))
}
_ => {
warn!("TODO handle {ca_st:?}");
warn!("TODO handle array {ca_st:?}");
return Err(Error::with_msg_no_trace(format!(
"can not yet handle type array {ca_st:?}"
)));
@@ -687,10 +752,12 @@ pub struct CaProto {
buf: NetBuf,
outbuf: NetBuf,
out: VecDeque<CaMsg>,
array_truncate: usize,
logged_proto_error_for_cid: BTreeMap<u32, bool>,
}
impl CaProto {
pub fn new(tcp: TcpStream, remote_addr_dbg: SocketAddrV4) -> Self {
pub fn new(tcp: TcpStream, remote_addr_dbg: SocketAddrV4, array_truncate: usize) -> Self {
Self {
tcp,
remote_addr_dbg,
@@ -698,6 +765,8 @@ impl CaProto {
buf: NetBuf::new(1024 * 128),
outbuf: NetBuf::new(1024 * 128),
out: VecDeque::new(),
array_truncate,
logged_proto_error_for_cid: BTreeMap::new(),
}
}
@@ -855,13 +924,22 @@ impl CaProto {
break match &self.state {
CaState::StdHead => {
let hi = HeadInfo::from_netbuf(&mut self.buf)?;
if hi.cmdid == 6
|| hi.cmdid > 26
|| hi.data_type > 10
|| hi.data_count > 4096
|| hi.payload_size > 1024 * 32
{
warn!("StdHead sees {hi:?}");
if hi.cmdid == 1 || hi.cmdid == 15 {
let sid = hi.param1;
if hi.payload_size == 0xffff && hi.data_count == 0 {
} else if hi.payload_size > 16368 {
if self.logged_proto_error_for_cid.contains_key(&sid) {
// TODO emit this as Item so that downstream can translate SID to name.
warn!(
"Protocol error payload_size 0x{:04x} data_count 0x{:04x} hi {:?}",
hi.payload_size, hi.data_count, hi
);
self.logged_proto_error_for_cid.insert(sid, true);
}
}
}
if hi.cmdid > 26 {
warn!("Enexpected cmdid {hi:?}");
}
if hi.payload_size == 0xffff && hi.data_count == 0 {
self.state = CaState::ExtHead(hi);
@@ -869,7 +947,7 @@ impl CaProto {
} else {
if hi.payload_size == 0 {
self.state = CaState::StdHead;
let msg = CaMsg::from_proto_infos(&hi, &[])?;
let msg = CaMsg::from_proto_infos(&hi, &[], self.array_truncate)?;
Ok(Some(CaItem::Msg(msg)))
} else {
self.state = CaState::Payload(hi);
@@ -880,9 +958,18 @@ impl CaProto {
CaState::ExtHead(hi) => {
let payload_size = self.buf.read_u32_be()?;
let data_count = self.buf.read_u32_be()?;
warn!("ExtHead payload_size {payload_size} data_count {data_count}");
if payload_size == 0 {
let msg = CaMsg::from_proto_infos(hi, &[])?;
if payload_size > 1024 * 256 {
warn!(
"ExtHead data_type {} payload_size {payload_size} data_count {data_count}",
hi.data_type
);
}
if payload_size <= 16368 {
warn!(
"ExtHead data_type {} payload_size {payload_size} data_count {data_count}",
hi.data_type
);
let msg = CaMsg::from_proto_infos(hi, &[], self.array_truncate)?;
self.state = CaState::StdHead;
Ok(Some(CaItem::Msg(msg)))
} else {
@@ -892,7 +979,7 @@ impl CaProto {
}
CaState::Payload(hi) => {
let g = self.buf.read_bytes(hi.payload_size as _)?;
let msg = CaMsg::from_proto_infos(hi, g)?;
let msg = CaMsg::from_proto_infos(hi, g, self.array_truncate)?;
self.state = CaState::StdHead;
Ok(Some(CaItem::Msg(msg)))
}
netfetch/src/ca/store.rs
+22 -1
View File
@@ -1,5 +1,6 @@
use crate::bsread::ChannelDescDecoded;
use crate::series::{Existence, SeriesId};
use crate::store::{CommonInsertQueue, CommonInsertQueueSender};
use async_channel::{Receiver, Sender};
use err::Error;
use scylla::prepared_statement::PreparedStatement;
@@ -48,13 +49,20 @@ pub struct DataStore {
pub qu_insert_scalar_f64: Arc<PreparedStatement>,
pub qu_insert_scalar_string: Arc<PreparedStatement>,
pub qu_insert_array_i8: Arc<PreparedStatement>,
pub qu_insert_array_i16: Arc<PreparedStatement>,
pub qu_insert_array_i32: Arc<PreparedStatement>,
pub qu_insert_array_f32: Arc<PreparedStatement>,
pub qu_insert_array_f64: Arc<PreparedStatement>,
pub chan_reg: Arc<ChannelRegistry>,
pub ciqs: CommonInsertQueueSender,
}
impl DataStore {
pub async fn new(pg_client: Arc<PgClient>, scy: Arc<ScySession>) -> Result<Self, Error> {
pub async fn new(
pg_client: Arc<PgClient>,
scy: Arc<ScySession>,
ciqs: CommonInsertQueueSender,
) -> Result<Self, Error> {
let q = scy
.prepare("insert into series (part, series, ts_msp, scalar_type, shape_dims) values (?, ?, ?, ?, ?)")
.await
@@ -101,6 +109,16 @@ impl DataStore {
.await
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
let qu_insert_array_i8 = Arc::new(q);
let q = scy
.prepare("insert into events_array_i16 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?)")
.await
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
let qu_insert_array_i16 = Arc::new(q);
let q = scy
.prepare("insert into events_array_i32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?)")
.await
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
let qu_insert_array_i32 = Arc::new(q);
let q = scy
.prepare("insert into events_array_f32 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?)")
.await
@@ -123,8 +141,11 @@ impl DataStore {
qu_insert_scalar_f64,
qu_insert_scalar_string,
qu_insert_array_i8,
qu_insert_array_i16,
qu_insert_array_i32,
qu_insert_array_f32,
qu_insert_array_f64,
ciqs,
};
Ok(ret)
}