Move workspace crates into subfolder
This commit is contained in:
Dominik Werder
106
crates/httpret/src/api4/binned.rs
Normal file
106
crates/httpret/src/api4/binned.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::ToPublicResponse;
|
||||
use crate::channelconfig::ch_conf_from_binned;
|
||||
use crate::err::Error;
|
||||
use crate::response_err;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::timeunits::SEC;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::APP_OCTET;
|
||||
use query::api4::binned::BinnedQuery;
|
||||
use tracing::Instrument;
|
||||
use url::Url;
|
||||
|
||||
async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
debug!("httpret plain_events_json req: {:?}", req);
|
||||
let (_head, _body) = req.into_parts();
|
||||
let query = BinnedQuery::from_url(&url).map_err(|e| {
|
||||
error!("binned_json: {e:?}");
|
||||
let msg = format!("can not parse query: {}", e.msg());
|
||||
e.add_public_msg(msg)
|
||||
})?;
|
||||
// TODO handle None case better and return 404
|
||||
let ch_conf = ch_conf_from_binned(&query, node_config)
|
||||
.await?
|
||||
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
|
||||
let span1 = span!(
|
||||
Level::INFO,
|
||||
"httpret::binned",
|
||||
beg = query.range().beg_u64() / SEC,
|
||||
end = query.range().end_u64() / SEC,
|
||||
ch = query.channel().name().clone(),
|
||||
);
|
||||
span1.in_scope(|| {
|
||||
debug!("begin");
|
||||
});
|
||||
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, node_config.node_config.cluster.clone())
|
||||
.instrument(span1)
|
||||
.await?;
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(buf))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(ACCEPT_ALL, |k| k.to_str().unwrap_or(ACCEPT_ALL));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)
|
||||
.map_err(Error::from)
|
||||
.map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
|
||||
};
|
||||
if req
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map_or(false, |x| x.as_str().contains("DOERR"))
|
||||
{
|
||||
Err(Error::with_msg_no_trace("hidden message").add_public_msg("PublicMessage"))?;
|
||||
}
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
Ok(binned_json(url, req, node_config).await?)
|
||||
} else if accept == APP_OCTET {
|
||||
Ok(response_err(
|
||||
StatusCode::NOT_ACCEPTABLE,
|
||||
format!("binary binned data not yet available"),
|
||||
)?)
|
||||
} else {
|
||||
let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("Unsupported Accept: {:?}", accept))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BinnedHandler {}
|
||||
|
||||
impl BinnedHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/binned" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() != Method::GET {
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
match binned(req, node_config).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
warn!("BinnedHandler handle sees: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
110
crates/httpret/src/api4/databuffer_tools.rs
Normal file
110
crates/httpret/src/api4/databuffer_tools.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::BodyStream;
|
||||
use crate::response_err;
|
||||
use bytes::Bytes;
|
||||
use err::thiserror;
|
||||
use err::ToPublicError;
|
||||
use futures_util::Stream;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FindActiveError {
|
||||
#[error("HttpBadAccept")]
|
||||
HttpBadAccept,
|
||||
#[error("HttpBadUrl")]
|
||||
HttpBadUrl,
|
||||
#[error("{0}")]
|
||||
Error(Box<dyn ToPublicError>),
|
||||
#[error("{0}")]
|
||||
UrlError(#[from] url::ParseError),
|
||||
#[error("InternalError")]
|
||||
InternalError,
|
||||
}
|
||||
|
||||
impl ToPublicError for FindActiveError {
|
||||
fn to_public_error(&self) -> String {
|
||||
match self {
|
||||
FindActiveError::HttpBadAccept => format!("{self}"),
|
||||
FindActiveError::HttpBadUrl => format!("{self}"),
|
||||
FindActiveError::Error(e) => e.to_public_error(),
|
||||
FindActiveError::UrlError(e) => format!("can not parse url: {e}"),
|
||||
FindActiveError::InternalError => format!("{self}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FindActiveHandler {}
|
||||
|
||||
impl FindActiveHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/tools/databuffer/findActive" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, FindActiveError> {
|
||||
if req.method() != Method::GET {
|
||||
Ok(response(StatusCode::NOT_ACCEPTABLE)
|
||||
.body(Body::empty())
|
||||
.map_err(|_| FindActiveError::InternalError)?)
|
||||
} else {
|
||||
match Self::handle_req(req, ncc).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
error!("{e}");
|
||||
let res = response_err(StatusCode::NOT_ACCEPTABLE, e.to_public_error())
|
||||
.map_err(|_| FindActiveError::InternalError)?;
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_req(req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, FindActiveError> {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)?
|
||||
};
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
type _A = netpod::BodyStream;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(BodyStream::wrapped(Box::pin(DummyStream::new()), "find_active".into()))
|
||||
.map_err(|_| FindActiveError::InternalError)?)
|
||||
} else {
|
||||
Err(FindActiveError::HttpBadAccept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DummyStream {}
|
||||
|
||||
impl DummyStream {
|
||||
pub fn new() -> Self {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for DummyStream {
|
||||
type Item = Result<Bytes, crate::err::Error>;
|
||||
|
||||
fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context) -> std::task::Poll<Option<Self::Item>> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
113
crates/httpret/src/api4/events.rs
Normal file
113
crates/httpret/src/api4/events.rs
Normal file
@@ -0,0 +1,113 @@
|
||||
use crate::channelconfig::chconf_from_events_v1;
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use crate::response_err;
|
||||
use crate::BodyStream;
|
||||
use crate::ToPublicResponse;
|
||||
use futures_util::stream;
|
||||
use futures_util::TryStreamExt;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::APP_OCTET;
|
||||
use query::api4::events::PlainEventsQuery;
|
||||
use url::Url;
|
||||
|
||||
pub struct EventsHandler {}
|
||||
|
||||
impl EventsHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/events" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() != Method::GET {
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
match plain_events(req, node_config).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
error!("EventsHandler sees: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn plain_events(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)
|
||||
.map_err(Error::from)
|
||||
.map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
|
||||
};
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
Ok(plain_events_json(url, req, node_config).await?)
|
||||
} else if accept == APP_OCTET {
|
||||
Ok(plain_events_binary(url, req, node_config).await?)
|
||||
} else {
|
||||
let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("Unsupported Accept: {:?}", accept))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
async fn plain_events_binary(
|
||||
url: Url,
|
||||
req: Request<Body>,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
debug!("plain_events_binary req: {:?}", req);
|
||||
let query = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
|
||||
let ch_conf = chconf_from_events_v1(&query, node_config).await?;
|
||||
info!("plain_events_binary chconf_from_events_v1: {ch_conf:?}");
|
||||
let s = stream::iter([Ok::<_, Error>(String::from("TODO_PREBINNED_BINARY_STREAM"))]);
|
||||
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
|
||||
s.map_err(Error::from),
|
||||
format!("plain_events_binary"),
|
||||
))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn plain_events_json(
|
||||
url: Url,
|
||||
req: Request<Body>,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
info!("plain_events_json req: {:?}", req);
|
||||
let (_head, _body) = req.into_parts();
|
||||
let query = PlainEventsQuery::from_url(&url)?;
|
||||
info!("plain_events_json query {query:?}");
|
||||
// TODO handle None case better and return 404
|
||||
let ch_conf = chconf_from_events_v1(&query, node_config)
|
||||
.await
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
|
||||
info!("plain_events_json chconf_from_events_v1: {ch_conf:?}");
|
||||
let item = streams::plaineventsjson::plain_events_json(&query, ch_conf, &node_config.node_config.cluster).await;
|
||||
let item = match item {
|
||||
Ok(item) => item,
|
||||
Err(e) => {
|
||||
error!("got error from streams::plaineventsjson::plain_events_json {e:?}");
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(buf))?;
|
||||
Ok(ret)
|
||||
}
|
||||
61
crates/httpret/src/api4/search.rs
Normal file
61
crates/httpret/src/api4/search.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::ToPublicResponse;
|
||||
use crate::Error;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::ChannelSearchQuery;
|
||||
use netpod::ChannelSearchResult;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use url::Url;
|
||||
|
||||
pub async fn channel_search(req: Request<Body>, node_config: &NodeConfigCached) -> Result<ChannelSearchResult, Error> {
|
||||
let url = Url::parse(&format!("dummy://{}", req.uri()))?;
|
||||
let query = ChannelSearchQuery::from_url(&url)?;
|
||||
info!("search query: {:?}", query);
|
||||
let res = dbconn::search::search_channel(query, node_config).await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub struct ChannelSearchHandler {}
|
||||
|
||||
impl ChannelSearchHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/search/channel" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match channel_search(req, node_config).await {
|
||||
Ok(item) => {
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
Ok(response(StatusCode::OK).body(Body::from(buf))?)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("handle: got error from channel_search: {e:?}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
110
crates/httpret/src/api4/status.rs
Normal file
110
crates/httpret/src/api4/status.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::ReqCtx;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::NodeStatus;
|
||||
use netpod::NodeStatusArchiverAppliance;
|
||||
use netpod::TableSizes;
|
||||
use std::collections::VecDeque;
|
||||
use std::time::Duration;
|
||||
|
||||
#[allow(unused)]
|
||||
async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, Error> {
|
||||
let ret = dbconn::table_sizes(node_config).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub struct StatusNodesRecursive {}
|
||||
|
||||
impl StatusNodesRecursive {
|
||||
pub fn path() -> &'static str {
|
||||
"/api/4/private/status/nodes/recursive"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == Self::path() {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let res = tokio::time::timeout(Duration::from_millis(1200), self.status(req, ctx, node_config)).await;
|
||||
let res = match res {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
let e = Error::from(e).add_public_msg("see timeout");
|
||||
return Ok(crate::bodystream::ToPublicResponse::to_public_response(&e));
|
||||
}
|
||||
};
|
||||
match res {
|
||||
Ok(status) => {
|
||||
let body = serde_json::to_vec(&status)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(body))?;
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("StatusNodesRecursive sees: {e}");
|
||||
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn status(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<NodeStatus, Error> {
|
||||
let (_head, _body) = req.into_parts();
|
||||
let archiver_appliance_status = match node_config.node.archiver_appliance.as_ref() {
|
||||
Some(k) => {
|
||||
let mut st = Vec::new();
|
||||
for p in &k.data_base_paths {
|
||||
let _m = match tokio::fs::metadata(p).await {
|
||||
Ok(m) => m,
|
||||
Err(_e) => {
|
||||
st.push((p.into(), false));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let _ = match tokio::fs::read_dir(p).await {
|
||||
Ok(rd) => rd,
|
||||
Err(_e) => {
|
||||
st.push((p.into(), false));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
st.push((p.into(), true));
|
||||
}
|
||||
Some(NodeStatusArchiverAppliance { readable: st })
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let database_size = dbconn::database_size(node_config).await.map_err(|e| format!("{e:?}"));
|
||||
let ret = NodeStatus {
|
||||
name: format!("{}:{}", node_config.node.host, node_config.node.port),
|
||||
version: core::env!("CARGO_PKG_VERSION").into(),
|
||||
is_sf_databuffer: node_config.node.sf_databuffer.is_some(),
|
||||
is_archiver_engine: node_config.node.channel_archiver.is_some(),
|
||||
is_archiver_appliance: node_config.node.archiver_appliance.is_some(),
|
||||
database_size: Some(database_size),
|
||||
//table_sizes: Some(table_sizes(node_config).await.map_err(Into::into)),
|
||||
archiver_appliance_status,
|
||||
subs: VecDeque::new(),
|
||||
};
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user