daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Udate deps, backend list for plain node

Browse Source
This commit is contained in:
Dominik Werder
2024-06-12 13:43:28 +02:00
parent 437e6d0d76
commit 03e8ac7a70
35 changed files with 1459 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
crates/httpret/src/api4/accounting.rs
View File

@@ -19,10 +19,12 @@ use netpod::log::*;
use netpod::req_uri_to_url;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use netpod::Shape;
use query::api4::AccountingIngestedBytesQuery;
use query::api4::AccountingToplistQuery;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
pub struct AccountingIngestedBytes {}
@@ -87,7 +89,33 @@ impl AccountingIngestedBytes {
#[derive(Debug, Serialize, Deserialize)]
pub struct Toplist {
toplist: Vec<(String, u64, u64)>,
dim0: Vec<(String, u64, u64)>,
dim1: Vec<(String, u64, u64)>,
infos_count_total: usize,
infos_missing_count: usize,
top1_usage_len: usize,
scalar_count: usize,
wave_count: usize,
found: usize,
incomplete_count: usize,
mismatch_count: usize,
}
impl Toplist {
fn new() -> Self {
Self {
dim0: Vec::new(),
dim1: Vec::new(),
infos_count_total: 0,
infos_missing_count: 0,
top1_usage_len: 0,
scalar_count: 0,
wave_count: 0,
found: 0,
incomplete_count: 0,
mismatch_count: 0,
}
}
}
pub struct AccountingToplistCounts {}
@@ -135,6 +163,7 @@ impl AccountingToplistCounts {
_ctx: &ReqCtx,
ncc: &NodeConfigCached,
) -> Result<Toplist, Error> {
let list_len_max = qu.limit() as usize;
// TODO assumes that accounting data is in the LT keyspace
let scyco = ncc
.node_config
@@ -145,22 +174,74 @@ impl AccountingToplistCounts {
let pgconf = &ncc.node_config.cluster.database;
let (pg, pgjh) = dbconn::create_connection(&pgconf).await?;
let mut top1 = scyllaconn::accounting::toplist::read_ts(qu.ts().ns(), scy).await?;
top1.sort_by_bytes();
let mut ret = Toplist { toplist: Vec::new() };
let series_ids: Vec<_> = top1.usage().iter().take(qu.limit() as _).map(|x| x.0).collect();
let infos = dbconn::channelinfo::info_for_series_ids(&series_ids, &pg)
.await
.map_err(Error::from_to_string)?;
let mut it = top1.usage().iter();
for info in infos {
let h = it.next().ok_or_else(|| Error::with_msg_no_trace("logic error"))?;
if info.series != h.0 {
let e = Error::with_msg_no_trace(format!("mismatch {} != {}", info.series, h.0));
warn!("{e}");
return Err(e);
top1.sort_by_counts();
let mut ret = Toplist::new();
let top1_usage = top1.usage();
ret.top1_usage_len = top1_usage.len();
let usage_map_0: BTreeMap<u64, (u64, u64)> = top1_usage.iter().map(|x| (x.0, (x.1, x.2))).collect();
let mut usage_it = usage_map_0.iter();
loop {
let mut series_ids = Vec::new();
let mut usages = Vec::new();
while let Some(u) = usage_it.next() {
series_ids.push(*u.0);
usages.push(u.1.clone());
if series_ids.len() >= 200 {
break;
}
}
if series_ids.len() == 0 {
break;
}
let infos = dbconn::channelinfo::info_for_series_ids(&series_ids, &pg)
.await
.map_err(Error::from_to_string)?;
for (_series, info_res) in &infos {
if let Some(info) = info_res {
match &info.shape {
Shape::Scalar => {
ret.scalar_count += 1;
}
Shape::Wave(_) => {
ret.wave_count += 1;
}
_ => {}
}
}
}
if usages.len() > infos.len() {
ret.incomplete_count += usages.len() - infos.len();
}
if infos.len() > usages.len() {
ret.incomplete_count += infos.len() - usages.len();
}
for ((series2, info_res), usage) in infos.into_iter().zip(usages.into_iter()) {
if let Some(info) = info_res {
if series2 != info.series {
ret.mismatch_count += 1;
}
ret.infos_count_total += 1;
// if info.name == "SINSB04-RMOD:PULSE-I-WF" {
// ret.found += 1;
// }
match &info.shape {
Shape::Scalar => {
ret.dim0.push((info.name, usage.0, usage.1));
}
Shape::Wave(_) => {
ret.dim1.push((info.name, usage.0, usage.1));
}
Shape::Image(_, _) => {}
}
} else {
ret.infos_missing_count += 1;
}
}
ret.toplist.push((info.name, h.1, h.2));
}
ret.dim0.sort_by_cached_key(|x| u64::MAX - x.1);
ret.dim1.sort_by_cached_key(|x| u64::MAX - x.1);
ret.dim0.truncate(list_len_max);
ret.dim1.truncate(list_len_max);
Ok(ret)
}
}

55
crates/httpret/src/api4/backend.rs Normal file
View File

@@ -0,0 +1,55 @@
use crate::bodystream::response;
use crate::err::Error;
use crate::requests::accepts_json_or_all;
use http::Method;
use http::StatusCode;
use httpclient::body_empty;
use httpclient::body_string;
use httpclient::Requ;
use httpclient::StreamResponse;
use netpod::NodeConfigCached;
use netpod::ReqCtx;
use netpod::ServiceVersion;
use std::collections::BTreeMap;
pub struct BackendListHandler {}
impl BackendListHandler {
pub fn handler(req: &Requ) -> Option<Self> {
if req.uri().path() == "/api/4/backend/list" {
Some(Self {})
} else {
None
}
}
pub async fn handle(
&self,
req: Requ,
_ctx: &ReqCtx,
ncc: &NodeConfigCached,
_service_version: &ServiceVersion,
) -> Result<StreamResponse, Error> {
if req.method() == Method::GET {
if accepts_json_or_all(req.headers()) {
let mut list = Vec::new();
if let Some(g) = &ncc.node_config.cluster.announce_backends {
for j in g {
let mut map = BTreeMap::new();
map.insert("name", j.clone());
list.push(map);
}
}
let res = serde_json::json!({
"backends_available": list,
});
let body = serde_json::to_string(&res)?;
Ok(response(StatusCode::OK).body(body_string(body))?)
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?)
}
}
}

13
crates/httpret/src/api4/eventdata.rs
View File

@@ -16,6 +16,7 @@ use httpclient::StreamResponse;
use netpod::log::*;
use netpod::NodeConfigCached;
use std::sync::Arc;
use tracing::Instrument;
#[derive(Debug, ThisError)]
pub enum EventDataError {
@@ -84,7 +85,19 @@ impl EventDataHandler {
.await
.map_err(|_| EventDataError::InternalError)?;
let (evsubq,) = nodenet::conn::events_parse_input_query(frames).map_err(|_| EventDataError::QueryParse)?;
let logspan = if false {
tracing::Span::none()
} else if evsubq.log_level() == "trace" {
trace!("enable trace for handler");
tracing::span!(tracing::Level::INFO, "log_span_trace")
} else if evsubq.log_level() == "debug" {
debug!("enable debug for handler");
tracing::span!(tracing::Level::INFO, "log_span_debug")
} else {
tracing::Span::none()
};
let stream = nodenet::conn::create_response_bytes_stream(evsubq, shared_res.scyqueue.as_ref(), ncc)
.instrument(logspan)
.await
.map_err(|e| EventDataError::Error(Box::new(e)))?;
let ret = response(StatusCode::OK)

66
crates/httpret/src/api4/events.rs
View File

@@ -29,7 +29,7 @@ use netpod::NodeConfigCached;
use netpod::ReqCtx;
use nodenet::client::OpenBoxedBytesViaHttp;
use query::api4::events::PlainEventsQuery;
use url::Url;
use tracing::Instrument;
pub struct EventsHandler {}
@@ -52,7 +52,26 @@ impl EventsHandler {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(body_empty())?);
}
match plain_events(req, ctx, &shared_res.pgqueue, ncc).await {
let self_name = "handle";
let url = req_uri_to_url(req.uri())?;
let evq =
PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
debug!("{self_name} evq {evq:?}");
let logspan = if false {
tracing::Span::none()
} else if evq.log_level() == "trace" {
trace!("enable trace for handler");
tracing::span!(tracing::Level::INFO, "log_span_trace")
} else if evq.log_level() == "debug" {
debug!("enable debug for handler");
tracing::span!(tracing::Level::INFO, "log_span_debug")
} else {
tracing::Span::none()
};
match plain_events(req, evq, ctx, &shared_res.pgqueue, ncc)
.instrument(logspan)
.await
{
Ok(ret) => Ok(ret),
Err(e) => {
error!("EventsHandler sees: {e}");
@@ -64,17 +83,17 @@ impl EventsHandler {
async fn plain_events(
req: Requ,
evq: PlainEventsQuery,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri())?;
if accepts_cbor_framed(req.headers()) {
Ok(plain_events_cbor_framed(url, req, ctx, pgqueue, ncc).await?)
Ok(plain_events_cbor_framed(req, evq, ctx, pgqueue, ncc).await?)
} else if accepts_json_framed(req.headers()) {
Ok(plain_events_json_framed(url, req, ctx, pgqueue, ncc).await?)
Ok(plain_events_json_framed(req, evq, ctx, pgqueue, ncc).await?)
} else if accepts_json_or_all(req.headers()) {
Ok(plain_events_json(url, req, ctx, pgqueue, ncc).await?)
Ok(plain_events_json(req, evq, ctx, pgqueue, ncc).await?)
} else {
let ret = response_err_msg(StatusCode::NOT_ACCEPTABLE, format!("unsupported accept {:?}", req))?;
Ok(ret)
@@ -82,17 +101,16 @@ async fn plain_events(
}
async fn plain_events_cbor_framed(
url: Url,
req: Requ,
evq: PlainEventsQuery,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
debug!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventscbor::plain_events_cbor_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
use future::ready;
@@ -121,17 +139,16 @@ async fn plain_events_cbor_framed(
}
async fn plain_events_json_framed(
url: Url,
req: Requ,
evq: PlainEventsQuery,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
debug!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventsjson::plain_events_json_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
let stream = bytes_chunks_to_framed(stream);
@@ -140,33 +157,26 @@ async fn plain_events_json_framed(
}
async fn plain_events_json(
url: Url,
req: Requ,
evq: PlainEventsQuery,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let self_name = "plain_events_json";
info!("{self_name} req: {:?}", req);
debug!("{self_name} req: {:?}", req);
let (_head, _body) = req.into_parts();
let query = PlainEventsQuery::from_url(&url)?;
info!("{self_name} query {query:?}");
// TODO handle None case better and return 404
let ch_conf = chconf_from_events_quorum(&query, ctx, pgqueue, ncc)
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await
.map_err(Error::from)?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("{self_name} chconf_from_events_quorum: {ch_conf:?}");
debug!("{self_name} chconf_from_events_quorum: {ch_conf:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let item = streams::plaineventsjson::plain_events_json(
&query,
ch_conf,
ctx,
&ncc.node_config.cluster,
Box::pin(open_bytes),
)
.await;
info!("{self_name} returned {}", item.is_ok());
let item =
streams::plaineventsjson::plain_events_json(&evq, ch_conf, ctx, &ncc.node_config.cluster, Box::pin(open_bytes))
.await;
debug!("{self_name} returned {}", item.is_ok());
let item = match item {
Ok(item) => item,
Err(e) => {
@@ -175,7 +185,7 @@ async fn plain_events_json(
}
};
let ret = response(StatusCode::OK).body(ToJsonBody::from(&item).into_body())?;
info!("{self_name} response created");
debug!("{self_name} response created");
Ok(ret)
}