daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Find active channels and deliver values

Browse Source
This commit is contained in:
Dominik Werder
2022-06-03 16:57:59 +02:00
parent 775650c2d8
commit 3cd1b7a640
49 changed files with 1002 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
httpret/src/api1.rs
View File

@@ -930,6 +930,7 @@ impl Api1EventsBinaryHandler {
.map(|x| Channel {
backend: backend.into(),
name: x.clone(),
series: None,
})
.collect();
// TODO use a better stream protocol with built-in error delivery.

2
httpret/src/channelarchiver.rs
View File

@@ -270,6 +270,7 @@ impl BlockRefStream {
let channel = Channel {
backend: "".into(),
name: channel_name,
series: None,
//name: "ARIDI-PCT:CURRENT".into(),
};
use archapp_wrap::archapp::archeng;
@@ -335,6 +336,7 @@ impl BlockStream {
let channel = Channel {
backend: node_config.node_config.cluster.backend.clone(),
name: channel_name,
series: None,
};
use archapp_wrap::archapp::archeng;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), database.clone()).await?;

403
httpret/src/channelconfig.rs
View File

@@ -1,19 +1,89 @@
use std::collections::BTreeMap;
use crate::err::Error;
use crate::{response, ToPublicResponse};
use dbconn::create_connection;
use disk::binned::query::PreBinnedQuery;
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
use http::{Method, Request, Response, StatusCode};
use hyper::Body;
use netpod::log::*;
use netpod::{get_url_query_pairs, Channel, ChannelConfigQuery, FromUrl, ScalarType, ScyllaConfig, Shape};
use netpod::query::BinnedQuery;
use netpod::{get_url_query_pairs, Channel, ChannelConfigQuery, Database, FromUrl, ScalarType, ScyllaConfig, Shape};
use netpod::{ChannelConfigResponse, NodeConfigCached};
use netpod::{ACCEPT_ALL, APP_JSON};
use scylla::batch::Consistency;
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
use scylla::transport::errors::{NewSessionError as ScyNewSessionError, QueryError as ScyQueryError};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use url::Url;
pub struct ChConf {
pub scalar_type: ScalarType,
pub shape: Shape,
}
pub async fn chconf_from_events_binary(_q: &PlainEventsBinaryQuery, _conf: &NodeConfigCached) -> Result<ChConf, Error> {
err::todoval()
}
pub async fn chconf_from_events_json(q: &PlainEventsJsonQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
if q.channel().backend != ncc.node_config.cluster.backend {
warn!(
"Mismatched backend {} VS {}",
q.channel().backend,
ncc.node_config.cluster.backend
);
}
if let Some(_conf) = &ncc.node_config.cluster.scylla {
// This requires the series id.
let series = q
.channel()
.series
.ok_or_else(|| Error::with_msg_no_trace(format!("needs a series id")))?;
// TODO use a common already running worker pool for these queries:
let dbconf = &ncc.node_config.cluster.database;
let dburl = format!(
"postgresql://{}:{}@{}:{}/{}",
dbconf.user, dbconf.pass, dbconf.host, dbconf.port, dbconf.name
);
let (pgclient, pgconn) = tokio_postgres::connect(&dburl, tokio_postgres::NoTls)
.await
.err_conv()?;
tokio::spawn(pgconn);
let res = pgclient
.query(
"select scalar_type, shape_dims from series_by_channel where series = $1",
&[&(series as i64)],
)
.await
.err_conv()?;
if res.len() == 0 {
error!("can not find channel for series {series}");
err::todoval()
} else if res.len() > 1 {
error!("can not find channel for series {series}");
err::todoval()
} else {
let row = res.first().unwrap();
let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(0) as u8)?;
// TODO can I get a slice from psql driver?
let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(1))?;
let ret = ChConf { scalar_type, shape };
Ok(ret)
}
} else {
err::todoval()
}
}
pub async fn chconf_from_prebinned(_q: &PreBinnedQuery, _conf: &NodeConfigCached) -> Result<ChConf, Error> {
err::todoval()
}
pub async fn chconf_from_binned(_q: &BinnedQuery, _conf: &NodeConfigCached) -> Result<ChConf, Error> {
err::todoval()
}
pub struct ChannelConfigHandler {}
impl ChannelConfigHandler {
@@ -80,15 +150,25 @@ impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
}
}
impl<T> ErrConv<T> for Result<T, tokio_postgres::Error> {
fn err_conv(self) -> Result<T, Error> {
match self {
Ok(k) => Ok(k),
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
}
}
}
async fn config_from_scylla(
chq: ChannelConfigQuery,
scyco: &ScyllaConfig,
_pgconf: Database,
scyconf: ScyllaConfig,
_node_config: &NodeConfigCached,
) -> Result<ChannelConfigResponse, Error> {
// Find the "series" id.
let scy = scylla::SessionBuilder::new()
.known_nodes(&scyco.hosts)
.use_keyspace(&scyco.keyspace, true)
.known_nodes(&scyconf.hosts)
.use_keyspace(&scyconf.keyspace, true)
.default_consistency(Consistency::One)
.build()
.await
@@ -130,14 +210,9 @@ pub async fn channel_config(req: Request<Body>, node_config: &NodeConfigCached)
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelConfigQuery::from_url(&url)?;
info!("channel_config for q {q:?}");
let conf = if q.channel.backend == "scylla" {
let scyco = node_config
.node_config
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
config_from_scylla(q, scyco, node_config).await?
let conf = if let Some(scyco) = &node_config.node_config.cluster.scylla {
let pgconf = node_config.node_config.cluster.database.clone();
config_from_scylla(q, pgconf, scyco.clone(), node_config).await?
} else if let Some(conf) = &node_config.node.channel_archiver {
archapp_wrap::archapp::archeng::channel_config_from_db(&q, conf, &node_config.node_config.cluster.database)
.await?
@@ -326,10 +401,11 @@ impl ScyllaChannelsWithType {
.err_conv()?;
let mut list = Vec::new();
for row in res.rows_typed_or_empty::<(String, i64)>() {
let (channel_name, _series) = row.err_conv()?;
let (channel_name, series) = row.err_conv()?;
let ch = Channel {
backend: facility.into(),
name: channel_name,
series: Some(series as u64),
};
list.push(ch);
}
@@ -337,3 +413,300 @@ impl ScyllaChannelsWithType {
Ok(ret)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScyllaChannelsActiveQuery {
tsedge: u64,
shape_kind: u32,
scalar_type: ScalarType,
}
impl FromUrl for ScyllaChannelsActiveQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
let pairs = get_url_query_pairs(url);
let s = pairs
.get("tsedge")
.ok_or_else(|| Error::with_public_msg_no_trace("missing tsedge"))?;
let tsedge: u64 = s.parse()?;
let s = pairs
.get("shape_kind")
.ok_or_else(|| Error::with_public_msg_no_trace("missing shape_kind"))?;
let shape_kind: u32 = s.parse()?;
let s = pairs
.get("scalar_type")
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalar_type"))?;
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
info!("parsed scalar type inp: {s:?} val: {scalar_type:?}");
Ok(Self {
tsedge,
scalar_type,
shape_kind,
})
}
}
pub struct ScyllaChannelsActive {}
impl ScyllaChannelsActive {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/scylla/channels/active" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() == Method::GET {
let accept_def = APP_JSON;
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ScyllaChannelsActiveQuery::from_url(&url)?;
let res = self.get_channels(&q, node_config).await?;
let body = Body::from(serde_json::to_vec(&res)?);
Ok(response(StatusCode::OK).body(body)?)
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
}
async fn get_channels(
&self,
q: &ScyllaChannelsActiveQuery,
node_config: &NodeConfigCached,
) -> Result<Vec<u64>, Error> {
let scyco = node_config
.node_config
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
let scy = scylla::SessionBuilder::new()
.known_nodes(&scyco.hosts)
.use_keyspace(&scyco.keyspace, true)
.default_consistency(Consistency::One)
.build()
.await
.err_conv()?;
// Database stores tsedge/ts_msp in units of (10 sec), and we additionally map to the grid.
let tsedge = q.tsedge / 10 / (6 * 2) * (6 * 2);
info!(
"ScyllaChannelsActive::get_channels tsedge {} (10s) {} (s)",
tsedge,
tsedge * 10
);
let mut ret = Vec::new();
for part in 0..256 {
let mut res = scy
.query_iter(
"select series from series_by_ts_msp where part = ? and ts_msp = ? and shape_kind = ? and scalar_type = ?",
(part as i32, tsedge as i32, q.shape_kind as i32, q.scalar_type.to_scylla_i32()),
)
.await
.err_conv()?;
use futures_util::StreamExt;
while let Some(row) = res.next().await {
let row = row.err_conv()?;
let (series,): (i64,) = row.into_typed().err_conv()?;
if series == 1254561075907984640 {
info!(
"FOUND ACTIVE series {} part {} tsedge {} scalar_type {}",
series,
part,
tsedge,
q.scalar_type.to_scylla_i32()
);
}
ret.push(series as u64);
}
}
Ok(ret)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ChannelFromSeriesQuery {
series: u64,
}
impl FromUrl for ChannelFromSeriesQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
let pairs = get_url_query_pairs(url);
let s = pairs
.get("series")
.ok_or_else(|| Error::with_public_msg_no_trace("missing series"))?;
let series: u64 = s.parse()?;
Ok(Self { series })
}
}
#[derive(Clone, Debug, Serialize)]
pub struct ChannelFromSeriesResponse {
facility: String,
channel: String,
scalar_type: ScalarType,
shape: Shape,
// TODO need a unique representation of the agg kind in the registry.
agg_kind: u32,
}
pub struct ChannelFromSeries {}
impl ChannelFromSeries {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/channel/series" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() == Method::GET {
let accept_def = APP_JSON;
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelFromSeriesQuery::from_url(&url)?;
let res = self.get_data(&q, node_config).await?;
let body = Body::from(serde_json::to_vec(&res)?);
Ok(response(StatusCode::OK).body(body)?)
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
}
async fn get_data(
&self,
q: &ChannelFromSeriesQuery,
node_config: &NodeConfigCached,
) -> Result<ChannelFromSeriesResponse, Error> {
let series = q.series as i64;
//let pgconn = create_connection(&node_config.node_config.cluster.database).await?;
let dbconf = &node_config.node_config.cluster.database;
// TODO unify the database nodes
let uri = format!(
"postgresql://{}:{}@{}:{}/{}",
dbconf.user, dbconf.pass, dbconf.host, 5432, dbconf.name
);
let (pgclient, conn) = tokio_postgres::connect(&uri, tokio_postgres::NoTls).await.err_conv()?;
// TODO monitor connection drop.
let _cjh = tokio::spawn(async move {
if let Err(e) = conn.await {
error!("connection error: {}", e);
}
Ok::<_, Error>(())
});
let res = pgclient
.query(
"select facility, channel, scalar_type, shape_dims, agg_kind from series_by_channel where series = $1",
&[&series],
)
.await?;
let res = if let Some(row) = res.first() {
row
} else {
// TODO return code 204
return Err(Error::with_msg_no_trace("can not find series"));
};
let facility: String = res.get(0);
let channel: String = res.get(1);
let scalar_type: i32 = res.get(2);
// TODO check and document the format in the storage:
let scalar_type = ScalarType::from_dtype_index(scalar_type as u8)?;
let shape: Vec<i32> = res.get(3);
let shape = Shape::from_scylla_shape_dims(&shape)?;
let agg_kind: i32 = res.get(4);
// TODO method is called from_scylla_shape_dims but document that postgres uses the same format.
let ret = ChannelFromSeriesResponse {
facility,
channel,
scalar_type,
shape,
agg_kind: agg_kind as u32,
};
Ok(ret)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct IocForChannelQuery {
channel_regex: String,
}
impl FromUrl for IocForChannelQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
let pairs = get_url_query_pairs(url);
let channel_regex = pairs
.get("channel_regex")
.ok_or_else(|| Error::with_public_msg_no_trace("missing channel_regex"))?
.into();
Ok(Self { channel_regex })
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ChannelIoc {
channel: String,
ioc: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct IocForChannelRes {
channels: Vec<Channel>,
}
pub struct IocForChannel {}
impl IocForChannel {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/ioc/channel" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() == Method::GET {
let accept_def = APP_JSON;
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = IocForChannelQuery::from_url(&url)?;
let res = self.find(&q, node_config).await?;
let body = Body::from(serde_json::to_vec(&res)?);
Ok(response(StatusCode::OK).body(body)?)
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
}
async fn find(&self, q: &IocForChannelQuery, node_config: &NodeConfigCached) -> Result<IocForChannelRes, Error> {
// TODO implement lookup in postgres
let _ = q;
let _pgconn = create_connection(&node_config.node_config.cluster.database).await?;
let _facility = "scylla";
let ret = IocForChannelRes { channels: vec![] };
Ok(ret)
}
}

25
httpret/src/events.rs
View File

@@ -1,3 +1,4 @@
use crate::channelconfig::{chconf_from_events_binary, chconf_from_events_json};
use crate::err::Error;
use crate::{response, response_err, BodyStream, ToPublicResponse};
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
@@ -52,13 +53,23 @@ async fn plain_events_binary(req: Request<Body>, node_config: &NodeConfigCached)
debug!("httpret plain_events_binary req: {:?}", req);
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let query = PlainEventsBinaryQuery::from_url(&url)?;
let chconf = chconf_from_events_binary(&query, node_config).await?;
let op = disk::channelexec::PlainEvents::new(
query.channel().clone(),
query.range().clone(),
query.disk_io_buffer_size(),
node_config.clone(),
);
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), AggKind::Plain, node_config).await?;
let s = disk::channelexec::channel_exec(
op,
query.channel(),
query.range(),
chconf.scalar_type,
chconf.shape,
AggKind::Plain,
node_config,
)
.await?;
let s = s.map(|item| item.make_frame());
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
s.map_err(Error::from),
@@ -71,6 +82,7 @@ async fn plain_events_json(req: Request<Body>, node_config: &NodeConfigCached) -
info!("httpret plain_events_json req: {:?}", req);
let (head, _body) = req.into_parts();
let query = PlainEventsJsonQuery::from_request_head(&head)?;
let chconf = chconf_from_events_json(&query, node_config).await?;
let op = disk::channelexec::PlainEventsJson::new(
query.channel().clone(),
query.range().clone(),
@@ -80,7 +92,16 @@ async fn plain_events_json(req: Request<Body>, node_config: &NodeConfigCached) -
query.events_max().unwrap_or(u64::MAX),
query.do_log(),
);
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), AggKind::Plain, node_config).await?;
let s = disk::channelexec::channel_exec(
op,
query.channel(),
query.range(),
chconf.scalar_type,
chconf.shape,
AggKind::Plain,
node_config,
)
.await?;
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
s.map_err(Error::from),
format!("plain_events_json"),

4
httpret/src/evinfo.rs
View File

@@ -1,3 +1,4 @@
use crate::channelconfig::chconf_from_events_json;
use crate::err::Error;
use crate::response;
use bytes::Bytes;
@@ -73,6 +74,7 @@ impl EventInfoScan {
query: &PlainEventsJsonQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>, Error> {
let chconf = chconf_from_events_json(&query, node_config).await?;
let ret = channel_exec(
EvInfoFunc::new(
query.clone(),
@@ -82,6 +84,8 @@ impl EventInfoScan {
),
query.channel(),
query.range(),
chconf.scalar_type,
chconf.shape,
AggKind::Stats1,
node_config,
)

30
httpret/src/httpret.rs
View File

@@ -15,6 +15,7 @@ use crate::err::Error;
use crate::gather::gather_get_json;
use crate::pulsemap::UpdateTask;
use bytes::Bytes;
use channelconfig::{chconf_from_binned, ChConf};
use disk::binned::query::PreBinnedQuery;
use future::Future;
use futures_core::Stream;
@@ -206,7 +207,7 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
let ret = serde_json::json!({
"data_api_version": {
"major": 4u32,
"minor": 1u32,
"minor": 2u32,
"patch": 0u32,
},
});
@@ -228,6 +229,12 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ScyllaChannelsWithType::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::IocForChannel::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ScyllaChannelsActive::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ChannelFromSeries::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = events::EventsHandler::handler(&req) {
h.handle(req, &node_config).await
} else if path == "/api/4/binned" {
@@ -527,20 +534,25 @@ async fn binned_inner(req: Request<Body>, node_config: &NodeConfigCached) -> Res
let (head, _body) = req.into_parts();
let url = Url::parse(&format!("dummy:{}", head.uri))?;
let query = BinnedQuery::from_url(&url)?;
let chconf = chconf_from_binned(&query, node_config).await?;
let desc = format!("binned-BEG-{}-END-{}", query.range().beg / SEC, query.range().end / SEC);
let span1 = span!(Level::INFO, "httpret::binned", desc = &desc.as_str());
span1.in_scope(|| {
debug!("binned STARTING {:?}", query);
});
match head.headers.get(http::header::ACCEPT) {
Some(v) if v == APP_OCTET => binned_binary(query, node_config).await,
Some(v) if v == APP_JSON || v == ACCEPT_ALL => binned_json(query, node_config).await,
Some(v) if v == APP_OCTET => binned_binary(query, chconf, node_config).await,
Some(v) if v == APP_JSON || v == ACCEPT_ALL => binned_json(query, chconf, node_config).await,
_ => Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?),
}
}
async fn binned_binary(query: BinnedQuery, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_bytes_for_http(&query, node_config).await {
async fn binned_binary(
query: BinnedQuery,
chconf: ChConf,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_bytes_for_http(&query, chconf.scalar_type, chconf.shape, node_config).await {
Ok(s) => {
response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_binary")))?
}
@@ -556,8 +568,12 @@ async fn binned_binary(query: BinnedQuery, node_config: &NodeConfigCached) -> Re
Ok(ret)
}
async fn binned_json(query: BinnedQuery, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_json(&query, node_config).await {
async fn binned_json(
query: BinnedQuery,
chconf: ChConf,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_json(&query, chconf.scalar_type, chconf.shape, node_config).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_json")))?,
Err(e) => {
if query.report_error() {

2
httpret/src/proxy.rs
View File

@@ -310,6 +310,8 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
backend: c.backend.clone(),
description: String::new(),
name: c.name.clone(),
// TODO api 0 does not provide a series id
series: 0,
shape: vec![],
source: c.source.clone(),
ty: c.ty.clone(),