daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Find active channels and deliver values

Browse Source
This commit is contained in:
Dominik Werder
2022-06-03 16:57:59 +02:00
parent 775650c2d8
commit 3cd1b7a640
49 changed files with 1002 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
nodenet/src/conn.rs
View File

@@ -116,40 +116,38 @@ async fn events_conn_handler_inner_try(
return Err((Error::with_msg("json parse error"), netout))?;
}
};
debug!("--- got query evq {:?}", evq);
info!("events_conn_handler_inner_try evq {:?}", evq);
let mut p1: Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>> = if evq.channel.backend == "scylla" {
if node_config.node.access_scylla {
let scyco = node_config.node_config.cluster.scylla.as_ref().unwrap();
match make_scylla_stream(&evq, scyco).await {
let mut p1: Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>> =
if let Some(conf) = &node_config.node_config.cluster.scylla {
let scyco = conf;
let dbconf = node_config.node_config.cluster.database.clone();
match make_scylla_stream(&evq, scyco, dbconf).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else if let Some(aa) = &node_config.node.channel_archiver {
match archapp_wrap::archapp::archeng::pipe::make_event_pipe(&evq, node_config.clone(), aa.clone()).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else if let Some(aa) = &node_config.node.archiver_appliance {
match archapp_wrap::make_event_pipe(&evq, aa).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else {
Box::pin(futures_util::stream::empty())
}
} else if let Some(aa) = &node_config.node.channel_archiver {
match archapp_wrap::archapp::archeng::pipe::make_event_pipe(&evq, node_config.clone(), aa.clone()).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else if let Some(aa) = &node_config.node.archiver_appliance {
match archapp_wrap::make_event_pipe(&evq, aa).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else {
match evq.agg_kind {
AggKind::EventBlobs => match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
},
_ => match disk::raw::conn::make_event_pipe(&evq, node_config).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
},
}
};
match evq.agg_kind {
AggKind::EventBlobs => match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
},
_ => match disk::raw::conn::make_event_pipe(&evq, node_config).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
},
}
};
let mut buf_len_histo = HistoLog2::new(5);
while let Some(item) = p1.next().await {
let item = item.make_frame();

165
nodenet/src/scylla.rs
View File

@@ -6,7 +6,7 @@ use items::waveevents::WaveEvents;
use items::{Framable, RangeCompletableItem, StreamItem};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::{NanoRange, ScalarType, ScyllaConfig, Shape};
use netpod::{Channel, Database, NanoRange, ScalarType, ScyllaConfig, Shape};
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
use scylla::transport::errors::{NewSessionError as ScyNewSessionError, QueryError as ScyQueryError};
use scylla::Session as ScySession;
@@ -14,6 +14,7 @@ use std::collections::VecDeque;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio_postgres::Client as PgClient;
trait ErrConv<T> {
fn err_conv(self) -> Result<T, Error>;
@@ -46,6 +47,15 @@ impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
}
}
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}
macro_rules! impl_read_values_fut {
($fname:ident, $self:expr, $ts_msp:expr) => {{
let fut = $fname($self.series, $ts_msp, $self.range.clone(), $self.scy.clone());
@@ -101,14 +111,18 @@ impl ReadValues {
fn next(&mut self) -> bool {
if let Some(ts_msp) = self.ts_msp.pop_front() {
self.fut = self.make_fut(ts_msp);
self.fut = self.make_fut(ts_msp, self.ts_msp.len() > 1);
true
} else {
false
}
}
fn make_fut(&mut self, ts_msp: u64) -> Pin<Box<dyn Future<Output = Box<dyn Framable>> + Send>> {
fn make_fut(
&mut self,
ts_msp: u64,
_has_more_msp: bool,
) -> Pin<Box<dyn Future<Output = Box<dyn Framable>> + Send>> {
// TODO this also needs to differentiate on Shape.
let fut = match &self.shape {
Shape::Scalar => match &self.scalar_type {
@@ -137,7 +151,7 @@ impl ReadValues {
enum FrState {
New,
FindSeries(Pin<Box<dyn Future<Output = Result<(i64, ScalarType, Shape), Error>> + Send>>),
FindSeries(Pin<Box<dyn Future<Output = Result<(ScalarType, Shape), Error>> + Send>>),
FindMsp(Pin<Box<dyn Future<Output = Result<Vec<u64>, Error>> + Send>>),
ReadValues(ReadValues),
Done,
@@ -145,26 +159,30 @@ enum FrState {
pub struct ScyllaFramableStream {
state: FrState,
facility: String,
channel_name: String,
#[allow(unused)]
evq: RawEventsQuery,
#[allow(unused)]
channel: Channel,
series: u64,
range: NanoRange,
scalar_type: Option<ScalarType>,
shape: Option<Shape>,
series: i64,
scy: Arc<ScySession>,
pgclient: Arc<PgClient>,
}
impl ScyllaFramableStream {
pub fn new(evq: &RawEventsQuery, scy: Arc<ScySession>) -> Self {
pub fn new(evq: &RawEventsQuery, scy: Arc<ScySession>, pgclient: Arc<PgClient>) -> Self {
Self {
state: FrState::New,
facility: evq.channel.backend.clone(),
channel_name: evq.channel.name().into(),
series: evq.channel.series.unwrap(),
evq: evq.clone(),
channel: evq.channel.clone(),
range: evq.range.clone(),
scalar_type: None,
shape: None,
series: 0,
scy,
pgclient,
}
}
}
@@ -177,18 +195,17 @@ impl Stream for ScyllaFramableStream {
loop {
break match self.state {
FrState::New => {
let fut = find_series(self.facility.clone(), self.channel_name.clone(), self.scy.clone());
let fut = find_series(self.series, self.pgclient.clone());
let fut = Box::pin(fut);
self.state = FrState::FindSeries(fut);
continue;
}
FrState::FindSeries(ref mut fut) => match fut.poll_unpin(cx) {
Ready(Ok((series, scalar_type, shape))) => {
info!("ScyllaFramableStream found series {}", series);
self.series = series;
Ready(Ok((scalar_type, shape))) => {
info!("ScyllaFramableStream found series {:?} {:?}", scalar_type, shape);
self.scalar_type = Some(scalar_type);
self.shape = Some(shape);
let fut = find_ts_msp(series, self.range.clone(), self.scy.clone());
let fut = find_ts_msp(self.series as i64, self.range.clone(), self.scy.clone());
let fut = Box::pin(fut);
self.state = FrState::FindMsp(fut);
continue;
@@ -206,10 +223,11 @@ impl Stream for ScyllaFramableStream {
info!("found ts_msp {ts_msp:?}");
// TODO get rid of into() for VecDeque
let mut st = ReadValues::new(
self.series,
self.series as i64,
self.scalar_type.as_ref().unwrap().clone(),
self.shape.as_ref().unwrap().clone(),
self.range.clone(),
// TODO get rid of the conversion:
ts_msp.into(),
self.scy.clone(),
);
@@ -245,25 +263,19 @@ impl Stream for ScyllaFramableStream {
}
}
async fn find_series(
facility: String,
channel_name: String,
scy: Arc<ScySession>,
) -> Result<(i64, ScalarType, Shape), Error> {
info!("find_series");
let res = {
let cql =
"select series, scalar_type, shape_dims from series_by_channel where facility = ? and channel_name = ?";
scy.query(cql, (&facility, &channel_name)).await.err_conv()?
async fn find_series(series: u64, pgclient: Arc<PgClient>) -> Result<(ScalarType, Shape), Error> {
info!("find_series series {}", series);
let rows = {
let q = "select facility, channel, scalar_type, shape_dims from series_by_channel where series = $1";
pgclient.query(q, &[&(series as i64)]).await.err_conv()?
};
let rows: Vec<_> = res.rows_typed_or_empty::<(i64, i32, Vec<i32>)>().collect();
if rows.len() > 1 {
error!("Multiple series found for channel, can not return data for ambiguous series");
if rows.len() < 1 {
return Err(Error::with_public_msg_no_trace(
"Multiple series found for channel, can not return data for ambiguous series",
));
}
if rows.len() < 1 {
if rows.len() > 1 {
error!("Multiple series found for channel, can not return data for ambiguous series");
return Err(Error::with_public_msg_no_trace(
"Multiple series found for channel, can not return data for ambiguous series",
));
@@ -271,28 +283,43 @@ async fn find_series(
let row = rows
.into_iter()
.next()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("can not find series for channel")))?
.err_conv()?;
info!("make_scylla_stream row {row:?}");
let series = row.0;
let scalar_type = ScalarType::from_scylla_i32(row.1)?;
let shape = Shape::from_scylla_shape_dims(&row.2)?;
.ok_or_else(|| Error::with_public_msg_no_trace(format!("can not find series for channel")))?;
info!("row {row:?}");
let _facility: String = row.get(0);
let _channel: String = row.get(1);
let a: i32 = row.get(2);
let scalar_type = ScalarType::from_scylla_i32(a)?;
let a: Vec<i32> = row.get(3);
let shape = Shape::from_scylla_shape_dims(&a)?;
info!("make_scylla_stream series {series} scalar_type {scalar_type:?} shape {shape:?}");
Ok((series, scalar_type, shape))
Ok((scalar_type, shape))
}
async fn find_ts_msp(series: i64, range: NanoRange, scy: Arc<ScySession>) -> Result<Vec<u64>, Error> {
info!("find_ts_msp series {} {:?}", series, range);
// TODO use prepared statements
let cql = "select ts_msp from ts_msp where series = ? and ts_msp < ? order by ts_msp desc limit 1";
let res = scy.query(cql, (series, range.beg as i64)).await.err_conv()?;
let mut before = vec![];
for row in res.rows_typed_or_empty::<(i64,)>() {
let row = row.err_conv()?;
before.push(row.0 as u64);
}
info!("FOUND BEFORE THE REQUESTED TIME: {} {:?}", before.len(), before);
let cql = "select ts_msp from ts_msp where series = ? and ts_msp >= ? and ts_msp < ?";
let res = scy
.query(cql, (series, range.beg as i64, range.end as i64))
.await
.err_conv()?;
let mut ret = vec![];
for x in before {
ret.push(x);
}
for row in res.rows_typed_or_empty::<(i64,)>() {
let row = row.err_conv()?;
ret.push(row.0 as u64);
}
info!("found in total {} rows", ret.len());
info!("found in total {} rows {:?}", ret.len(), ret);
Ok(ret)
}
@@ -307,34 +334,31 @@ macro_rules! read_next_scalar_values {
type ST = $st;
type SCYTY = $scyty;
info!("{} series {} ts_msp {}", stringify!($fname), series, ts_msp);
// TODO add the constraint on range!
warn!("remove the limit clause, add range check");
// TODO use typed timestamp..
let ts_lsp_max = if range.end < ts_msp {
// We should not be here anyway.
warn!("range.end < ts_msp");
0
} else {
range.end - ts_msp
};
let cql = concat!(
"select ts_lsp, pulse, value from ",
$table_name,
" where series = ? and ts_msp = ? and ts_lsp < ?"
" where series = ? and ts_msp = ?"
);
let res = scy
.query(cql, (series, ts_msp as i64, ts_lsp_max as i64))
.await
.err_conv()?;
let res = scy.query(cql, (series, ts_msp as i64)).await.err_conv()?;
let mut ret = ScalarEvents::<ST>::empty();
let mut discarded = 0;
for row in res.rows_typed_or_empty::<(i64, i64, SCYTY)>() {
let row = row.err_conv()?;
let ts = ts_msp + row.0 as u64;
let pulse = row.1 as u64;
let value = row.2 as ST;
ret.push(ts, pulse, value);
if ts < range.beg || ts >= range.end {
discarded += 1;
} else {
ret.push(ts, pulse, value);
}
}
info!("found in total {} events ts_msp {}", ret.tss.len(), ts_msp);
info!(
"found in total {} events ts_msp {} discarded {}",
ret.tss.len(),
ts_msp,
discarded
);
Ok(ret)
}
};
@@ -351,25 +375,12 @@ macro_rules! read_next_array_values {
type ST = $st;
type SCYTY = $scyty;
info!("{} series {} ts_msp {}", stringify!($fname), series, ts_msp);
// TODO add the constraint on range!
warn!("remove the limit clause, add range check");
// TODO use typed timestamp..
let ts_lsp_max = if range.end < ts_msp {
// We should not be here anyway.
warn!("range.end < ts_msp");
0
} else {
range.end - ts_msp
};
let cql = concat!(
"select ts_lsp, pulse, value from ",
$table_name,
" where series = ? and ts_msp = ? and ts_lsp < ?"
" where series = ? and ts_msp = ?"
);
let res = scy
.query(cql, (series, ts_msp as i64, ts_lsp_max as i64))
.await
.err_conv()?;
let res = scy.query(cql, (series, ts_msp as i64)).await.err_conv()?;
let mut ret = WaveEvents::<ST>::empty();
for row in res.rows_typed_or_empty::<(i64, i64, Vec<SCYTY>)>() {
let row = row.err_conv()?;
@@ -393,16 +404,26 @@ read_next_array_values!(read_next_values_array_u16, u16, i16, "events_wave_u16")
pub async fn make_scylla_stream(
evq: &RawEventsQuery,
scyco: &ScyllaConfig,
dbconf: Database,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error> {
info!("make_scylla_stream open scylla connection");
// TODO reuse existing connection:
let scy = scylla::SessionBuilder::new()
.known_nodes(&scyco.hosts)
.use_keyspace(&scyco.keyspace, true)
.build()
.await
.err_conv()?;
let u = {
let d = &dbconf;
format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, d.port, d.name)
};
let (pgclient, pgconn) = tokio_postgres::connect(&u, tokio_postgres::NoTls).await.err_conv()?;
// TODO use common connection/pool:
tokio::spawn(pgconn);
let pgclient = Arc::new(pgclient);
let scy = Arc::new(scy);
let res = Box::pin(ScyllaFramableStream::new(evq, scy)) as _;
let res = Box::pin(ScyllaFramableStream::new(evq, scy, pgclient)) as _;
Ok(res)
}