daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add json-framed encoding, docs, refactor

Browse Source
This commit is contained in:
Dominik Werder
2024-04-28 18:41:06 +02:00
parent b0eab82c93
commit 1b1e0f5a72
52 changed files with 1539 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
crates/httpret/src/api4/accounting.rs
View File

@@ -20,6 +20,9 @@ use netpod::req_uri_to_url;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use query::api4::AccountingIngestedBytesQuery;
use query::api4::AccountingToplistQuery;
use serde::Deserialize;
use serde::Serialize;
pub struct AccountingIngestedBytes {}
@@ -73,7 +76,7 @@ impl AccountingIngestedBytes {
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
let scy = scyllaconn::conn::create_scy_session(scyco).await?;
let mut stream = scyllaconn::accounting::AccountingStreamScylla::new(q.range().try_into()?, scy);
let mut stream = scyllaconn::accounting::totals::AccountingStreamScylla::new(q.range().try_into()?, scy);
let mut ret = AccountingEvents::empty();
while let Some(item) = stream.next().await {
let mut item = item?;
@@ -82,3 +85,83 @@ impl AccountingIngestedBytes {
Ok(ret)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Toplist {
toplist: Vec<(String, u64, u64)>,
}
pub struct AccountingToplistCounts {}
impl AccountingToplistCounts {
pub fn handler(req: &Requ) -> Option<Self> {
if req.uri().path().starts_with("/api/4/accounting/toplist/counts") {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Requ, ctx: &ReqCtx, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
if req.method() == Method::GET {
if accepts_json_or_all(req.headers()) {
match self.handle_get(req, ctx, ncc).await {
Ok(x) => Ok(x),
Err(e) => {
error!("{e}");
let e2 = e.to_public_error();
let s = serde_json::to_string(&e2)?;
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body_string(s))?)
}
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?)
}
}
async fn handle_get(&self, req: Requ, ctx: &ReqCtx, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri())?;
let qu = AccountingToplistQuery::from_url(&url)?;
let res = self.fetch_data(qu, ctx, ncc).await?;
let body = ToJsonBody::from(&res).into_body();
Ok(response(StatusCode::OK).body(body)?)
}
async fn fetch_data(
&self,
qu: AccountingToplistQuery,
_ctx: &ReqCtx,
ncc: &NodeConfigCached,
) -> Result<Toplist, Error> {
let scyco = ncc
.node_config
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
let scy = scyllaconn::conn::create_scy_session(scyco).await?;
let pgconf = &ncc.node_config.cluster.database;
let pg = dbconn::create_connection(&pgconf).await?;
let mut top1 = scyllaconn::accounting::toplist::read_ts(qu.ts().0, scy).await?;
top1.sort_by_bytes();
let mut ret = Toplist { toplist: Vec::new() };
let series_ids: Vec<_> = top1.usage().iter().take(qu.limit() as _).map(|x| x.0).collect();
let infos = dbconn::channelinfo::info_for_series_ids(&series_ids, &pg)
.await
.map_err(Error::from_to_string)?;
let mut it = top1.usage().iter();
for info in infos {
let h = it.next().ok_or_else(|| Error::with_msg_no_trace("logic error"))?;
if info.series != h.0 {
let e = Error::with_msg_no_trace(format!("mismatch {} != {}", info.series, h.0));
warn!("{e}");
return Err(e);
}
ret.toplist.push((info.name, h.1, h.2));
}
Ok(ret)
}
}

10
crates/httpret/src/api4/docs.rs
View File

@@ -72,6 +72,7 @@ fn extract_all_files() -> Contents {
}
}
// .
fn blob() -> &'static [u8] {
include_bytes!(concat!("../../../../apidoc/book.cbor"))
}
@@ -84,7 +85,7 @@ impl DocsHandler {
}
pub fn handler(req: &Requ) -> Option<Self> {
if req.uri().path().starts_with(Self::path_prefix()) {
if req.uri().path().starts_with(Self::path_prefix()) || req.uri().path().starts_with("/api/4/documentation") {
Some(Self {})
} else {
None
@@ -93,6 +94,13 @@ impl DocsHandler {
pub async fn handle(&self, req: Requ, _ctx: &ReqCtx) -> Result<StreamResponse, Error> {
let path = req.uri().path();
if path.starts_with("/api/4/documentation") {
let ret = http::Response::builder()
.status(StatusCode::TEMPORARY_REDIRECT)
.header(http::header::LOCATION, "/api/4/docs/")
.body(body_empty())?;
return Ok(ret);
}
if path == "/api/4/docs" {
let ret = http::Response::builder()
.status(StatusCode::TEMPORARY_REDIRECT)

83
crates/httpret/src/api4/events.rs
View File

@@ -1,7 +1,8 @@
use crate::bodystream::response_err_msg;
use crate::channelconfig::chconf_from_events_quorum;
use crate::err::Error;
use crate::requests::accepts_cbor_frames;
use crate::requests::accepts_cbor_framed;
use crate::requests::accepts_json_framed;
use crate::requests::accepts_json_or_all;
use crate::response;
use crate::ToPublicResponse;
@@ -9,6 +10,7 @@ use bytes::Bytes;
use bytes::BytesMut;
use futures_util::future;
use futures_util::stream;
use futures_util::Stream;
use futures_util::StreamExt;
use http::Method;
use http::StatusCode;
@@ -59,8 +61,10 @@ impl EventsHandler {
async fn plain_events(req: Requ, ctx: &ReqCtx, node_config: &NodeConfigCached) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri())?;
if accepts_cbor_frames(req.headers()) {
Ok(plain_events_cbor(url, req, ctx, node_config).await?)
if accepts_cbor_framed(req.headers()) {
Ok(plain_events_cbor_framed(url, req, ctx, node_config).await?)
} else if accepts_json_framed(req.headers()) {
Ok(plain_events_json_framed(url, req, ctx, node_config).await?)
} else if accepts_json_or_all(req.headers()) {
Ok(plain_events_json(url, req, ctx, node_config).await?)
} else {
@@ -69,32 +73,62 @@ async fn plain_events(req: Requ, ctx: &ReqCtx, node_config: &NodeConfigCached) -
}
}
async fn plain_events_cbor(url: Url, req: Requ, ctx: &ReqCtx, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
async fn plain_events_cbor_framed(
url: Url,
req: Requ,
ctx: &ReqCtx,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
let ch_conf = chconf_from_events_quorum(&evq, ctx, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_cbor chconf_from_events_quorum: {ch_conf:?} {req:?}");
info!("plain_events_cbor_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventscbor::plain_events_cbor(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
let stream = streams::plaineventscbor::plain_events_cbor_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
use future::ready;
let stream = stream
.flat_map(|x| match x {
Ok(y) => {
use bytes::BufMut;
let buf = y.into_inner();
let mut b2 = BytesMut::with_capacity(8);
let adv = (buf.len() + 7) / 8 * 8;
let pad = adv - buf.len();
let mut b2 = BytesMut::with_capacity(16);
b2.put_u32_le(buf.len() as u32);
stream::iter([Ok::<_, Error>(b2.freeze()), Ok(buf)])
b2.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
let mut b3 = BytesMut::with_capacity(16);
b3.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0][..pad]);
stream::iter([Ok::<_, Error>(b2.freeze()), Ok(buf), Ok(b3.freeze())])
}
Err(e) => {
let e = Error::with_msg_no_trace(e.to_string());
stream::iter([Err(e), Ok(Bytes::new()), Ok(Bytes::new())])
}
// TODO handle other cases
_ => stream::iter([Ok(Bytes::new()), Ok(Bytes::new())]),
})
.filter(|x| if let Ok(x) = x { ready(x.len() > 0) } else { ready(true) });
let ret = response(StatusCode::OK).body(body_stream(stream))?;
Ok(ret)
}
async fn plain_events_json_framed(
url: Url,
req: Requ,
ctx: &ReqCtx,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
let ch_conf = chconf_from_events_quorum(&evq, ctx, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_json_framed chconf_from_events_quorum: {ch_conf:?} {req:?}");
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventsjson::plain_events_json_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
let stream = bytes_chunks_to_framed(stream);
let ret = response(StatusCode::OK).body(body_stream(stream))?;
Ok(ret)
}
async fn plain_events_json(
url: Url,
req: Requ,
@@ -133,3 +167,32 @@ async fn plain_events_json(
info!("{self_name} response created");
Ok(ret)
}
fn bytes_chunks_to_framed<S, T>(stream: S) -> impl Stream<Item = Result<Bytes, Error>>
where
S: Stream<Item = Result<T, err::Error>>,
T: Into<Bytes>,
{
use future::ready;
stream
// TODO unify this map to padded bytes for both json and cbor output
.flat_map(|x| match x {
Ok(y) => {
use bytes::BufMut;
let buf = y.into();
let adv = (buf.len() + 7) / 8 * 8;
let pad = adv - buf.len();
let mut b2 = BytesMut::with_capacity(16);
b2.put_u32_le(buf.len() as u32);
b2.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
let mut b3 = BytesMut::with_capacity(16);
b3.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0][..pad]);
stream::iter([Ok::<_, Error>(b2.freeze()), Ok(buf), Ok(b3.freeze())])
}
Err(e) => {
let e = Error::with_msg_no_trace(e.to_string());
stream::iter([Err(e), Ok(Bytes::new()), Ok(Bytes::new())])
}
})
.filter(|x| if let Ok(x) = x { ready(x.len() > 0) } else { ready(true) })
}