Move workspace crates into subfolder
This commit is contained in:
Dominik Werder
1010
crates/httpret/src/api1.rs
Normal file
1010
crates/httpret/src/api1.rs
Normal file
File diff suppressed because it is too large
Load Diff
5
crates/httpret/src/api4.rs
Normal file
5
crates/httpret/src/api4.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
pub mod binned;
|
||||
pub mod databuffer_tools;
|
||||
pub mod events;
|
||||
pub mod search;
|
||||
pub mod status;
|
||||
106
crates/httpret/src/api4/binned.rs
Normal file
106
crates/httpret/src/api4/binned.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::ToPublicResponse;
|
||||
use crate::channelconfig::ch_conf_from_binned;
|
||||
use crate::err::Error;
|
||||
use crate::response_err;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::timeunits::SEC;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::APP_OCTET;
|
||||
use query::api4::binned::BinnedQuery;
|
||||
use tracing::Instrument;
|
||||
use url::Url;
|
||||
|
||||
async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
debug!("httpret plain_events_json req: {:?}", req);
|
||||
let (_head, _body) = req.into_parts();
|
||||
let query = BinnedQuery::from_url(&url).map_err(|e| {
|
||||
error!("binned_json: {e:?}");
|
||||
let msg = format!("can not parse query: {}", e.msg());
|
||||
e.add_public_msg(msg)
|
||||
})?;
|
||||
// TODO handle None case better and return 404
|
||||
let ch_conf = ch_conf_from_binned(&query, node_config)
|
||||
.await?
|
||||
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
|
||||
let span1 = span!(
|
||||
Level::INFO,
|
||||
"httpret::binned",
|
||||
beg = query.range().beg_u64() / SEC,
|
||||
end = query.range().end_u64() / SEC,
|
||||
ch = query.channel().name().clone(),
|
||||
);
|
||||
span1.in_scope(|| {
|
||||
debug!("begin");
|
||||
});
|
||||
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, node_config.node_config.cluster.clone())
|
||||
.instrument(span1)
|
||||
.await?;
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(buf))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(ACCEPT_ALL, |k| k.to_str().unwrap_or(ACCEPT_ALL));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)
|
||||
.map_err(Error::from)
|
||||
.map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
|
||||
};
|
||||
if req
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map_or(false, |x| x.as_str().contains("DOERR"))
|
||||
{
|
||||
Err(Error::with_msg_no_trace("hidden message").add_public_msg("PublicMessage"))?;
|
||||
}
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
Ok(binned_json(url, req, node_config).await?)
|
||||
} else if accept == APP_OCTET {
|
||||
Ok(response_err(
|
||||
StatusCode::NOT_ACCEPTABLE,
|
||||
format!("binary binned data not yet available"),
|
||||
)?)
|
||||
} else {
|
||||
let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("Unsupported Accept: {:?}", accept))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BinnedHandler {}
|
||||
|
||||
impl BinnedHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/binned" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() != Method::GET {
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
match binned(req, node_config).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
warn!("BinnedHandler handle sees: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
110
crates/httpret/src/api4/databuffer_tools.rs
Normal file
110
crates/httpret/src/api4/databuffer_tools.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::BodyStream;
|
||||
use crate::response_err;
|
||||
use bytes::Bytes;
|
||||
use err::thiserror;
|
||||
use err::ToPublicError;
|
||||
use futures_util::Stream;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FindActiveError {
|
||||
#[error("HttpBadAccept")]
|
||||
HttpBadAccept,
|
||||
#[error("HttpBadUrl")]
|
||||
HttpBadUrl,
|
||||
#[error("{0}")]
|
||||
Error(Box<dyn ToPublicError>),
|
||||
#[error("{0}")]
|
||||
UrlError(#[from] url::ParseError),
|
||||
#[error("InternalError")]
|
||||
InternalError,
|
||||
}
|
||||
|
||||
impl ToPublicError for FindActiveError {
|
||||
fn to_public_error(&self) -> String {
|
||||
match self {
|
||||
FindActiveError::HttpBadAccept => format!("{self}"),
|
||||
FindActiveError::HttpBadUrl => format!("{self}"),
|
||||
FindActiveError::Error(e) => e.to_public_error(),
|
||||
FindActiveError::UrlError(e) => format!("can not parse url: {e}"),
|
||||
FindActiveError::InternalError => format!("{self}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FindActiveHandler {}
|
||||
|
||||
impl FindActiveHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/tools/databuffer/findActive" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, FindActiveError> {
|
||||
if req.method() != Method::GET {
|
||||
Ok(response(StatusCode::NOT_ACCEPTABLE)
|
||||
.body(Body::empty())
|
||||
.map_err(|_| FindActiveError::InternalError)?)
|
||||
} else {
|
||||
match Self::handle_req(req, ncc).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
error!("{e}");
|
||||
let res = response_err(StatusCode::NOT_ACCEPTABLE, e.to_public_error())
|
||||
.map_err(|_| FindActiveError::InternalError)?;
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_req(req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, FindActiveError> {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)?
|
||||
};
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
type _A = netpod::BodyStream;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(BodyStream::wrapped(Box::pin(DummyStream::new()), "find_active".into()))
|
||||
.map_err(|_| FindActiveError::InternalError)?)
|
||||
} else {
|
||||
Err(FindActiveError::HttpBadAccept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DummyStream {}
|
||||
|
||||
impl DummyStream {
|
||||
pub fn new() -> Self {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for DummyStream {
|
||||
type Item = Result<Bytes, crate::err::Error>;
|
||||
|
||||
fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context) -> std::task::Poll<Option<Self::Item>> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
113
crates/httpret/src/api4/events.rs
Normal file
113
crates/httpret/src/api4/events.rs
Normal file
@@ -0,0 +1,113 @@
|
||||
use crate::channelconfig::chconf_from_events_v1;
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use crate::response_err;
|
||||
use crate::BodyStream;
|
||||
use crate::ToPublicResponse;
|
||||
use futures_util::stream;
|
||||
use futures_util::TryStreamExt;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::APP_OCTET;
|
||||
use query::api4::events::PlainEventsQuery;
|
||||
use url::Url;
|
||||
|
||||
pub struct EventsHandler {}
|
||||
|
||||
impl EventsHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/events" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() != Method::GET {
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
match plain_events(req, node_config).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
error!("EventsHandler sees: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn plain_events(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
let url = {
|
||||
let s1 = format!("dummy:{}", req.uri());
|
||||
Url::parse(&s1)
|
||||
.map_err(Error::from)
|
||||
.map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
|
||||
};
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
Ok(plain_events_json(url, req, node_config).await?)
|
||||
} else if accept == APP_OCTET {
|
||||
Ok(plain_events_binary(url, req, node_config).await?)
|
||||
} else {
|
||||
let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("Unsupported Accept: {:?}", accept))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
async fn plain_events_binary(
|
||||
url: Url,
|
||||
req: Request<Body>,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
debug!("plain_events_binary req: {:?}", req);
|
||||
let query = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
|
||||
let ch_conf = chconf_from_events_v1(&query, node_config).await?;
|
||||
info!("plain_events_binary chconf_from_events_v1: {ch_conf:?}");
|
||||
let s = stream::iter([Ok::<_, Error>(String::from("TODO_PREBINNED_BINARY_STREAM"))]);
|
||||
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
|
||||
s.map_err(Error::from),
|
||||
format!("plain_events_binary"),
|
||||
))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn plain_events_json(
|
||||
url: Url,
|
||||
req: Request<Body>,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
info!("plain_events_json req: {:?}", req);
|
||||
let (_head, _body) = req.into_parts();
|
||||
let query = PlainEventsQuery::from_url(&url)?;
|
||||
info!("plain_events_json query {query:?}");
|
||||
// TODO handle None case better and return 404
|
||||
let ch_conf = chconf_from_events_v1(&query, node_config)
|
||||
.await
|
||||
.map_err(Error::from)?
|
||||
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
|
||||
info!("plain_events_json chconf_from_events_v1: {ch_conf:?}");
|
||||
let item = streams::plaineventsjson::plain_events_json(&query, ch_conf, &node_config.node_config.cluster).await;
|
||||
let item = match item {
|
||||
Ok(item) => item,
|
||||
Err(e) => {
|
||||
error!("got error from streams::plaineventsjson::plain_events_json {e:?}");
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(buf))?;
|
||||
Ok(ret)
|
||||
}
|
||||
61
crates/httpret/src/api4/search.rs
Normal file
61
crates/httpret/src/api4/search.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::bodystream::ToPublicResponse;
|
||||
use crate::Error;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::ChannelSearchQuery;
|
||||
use netpod::ChannelSearchResult;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use url::Url;
|
||||
|
||||
pub async fn channel_search(req: Request<Body>, node_config: &NodeConfigCached) -> Result<ChannelSearchResult, Error> {
|
||||
let url = Url::parse(&format!("dummy://{}", req.uri()))?;
|
||||
let query = ChannelSearchQuery::from_url(&url)?;
|
||||
info!("search query: {:?}", query);
|
||||
let res = dbconn::search::search_channel(query, node_config).await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub struct ChannelSearchHandler {}
|
||||
|
||||
impl ChannelSearchHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/search/channel" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match channel_search(req, node_config).await {
|
||||
Ok(item) => {
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
Ok(response(StatusCode::OK).body(Body::from(buf))?)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("handle: got error from channel_search: {e:?}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
110
crates/httpret/src/api4/status.rs
Normal file
110
crates/httpret/src/api4/status.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::ReqCtx;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::NodeStatus;
|
||||
use netpod::NodeStatusArchiverAppliance;
|
||||
use netpod::TableSizes;
|
||||
use std::collections::VecDeque;
|
||||
use std::time::Duration;
|
||||
|
||||
#[allow(unused)]
|
||||
async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, Error> {
|
||||
let ret = dbconn::table_sizes(node_config).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub struct StatusNodesRecursive {}
|
||||
|
||||
impl StatusNodesRecursive {
|
||||
pub fn path() -> &'static str {
|
||||
"/api/4/private/status/nodes/recursive"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == Self::path() {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let res = tokio::time::timeout(Duration::from_millis(1200), self.status(req, ctx, node_config)).await;
|
||||
let res = match res {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
let e = Error::from(e).add_public_msg("see timeout");
|
||||
return Ok(crate::bodystream::ToPublicResponse::to_public_response(&e));
|
||||
}
|
||||
};
|
||||
match res {
|
||||
Ok(status) => {
|
||||
let body = serde_json::to_vec(&status)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(body))?;
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("StatusNodesRecursive sees: {e}");
|
||||
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn status(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<NodeStatus, Error> {
|
||||
let (_head, _body) = req.into_parts();
|
||||
let archiver_appliance_status = match node_config.node.archiver_appliance.as_ref() {
|
||||
Some(k) => {
|
||||
let mut st = Vec::new();
|
||||
for p in &k.data_base_paths {
|
||||
let _m = match tokio::fs::metadata(p).await {
|
||||
Ok(m) => m,
|
||||
Err(_e) => {
|
||||
st.push((p.into(), false));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let _ = match tokio::fs::read_dir(p).await {
|
||||
Ok(rd) => rd,
|
||||
Err(_e) => {
|
||||
st.push((p.into(), false));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
st.push((p.into(), true));
|
||||
}
|
||||
Some(NodeStatusArchiverAppliance { readable: st })
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let database_size = dbconn::database_size(node_config).await.map_err(|e| format!("{e:?}"));
|
||||
let ret = NodeStatus {
|
||||
name: format!("{}:{}", node_config.node.host, node_config.node.port),
|
||||
version: core::env!("CARGO_PKG_VERSION").into(),
|
||||
is_sf_databuffer: node_config.node.sf_databuffer.is_some(),
|
||||
is_archiver_engine: node_config.node.channel_archiver.is_some(),
|
||||
is_archiver_appliance: node_config.node.archiver_appliance.is_some(),
|
||||
database_size: Some(database_size),
|
||||
//table_sizes: Some(table_sizes(node_config).await.map_err(Into::into)),
|
||||
archiver_appliance_status,
|
||||
subs: VecDeque::new(),
|
||||
};
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
126
crates/httpret/src/bodystream.rs
Normal file
126
crates/httpret/src/bodystream.rs
Normal file
@@ -0,0 +1,126 @@
|
||||
use crate::err::Error;
|
||||
use bytes::Bytes;
|
||||
use futures_util::{Stream, StreamExt};
|
||||
use http::HeaderMap;
|
||||
use http::{Response, StatusCode};
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::APP_JSON;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tracing::field::Empty;
|
||||
use tracing::{span, Level};
|
||||
|
||||
pub fn response<T>(status: T) -> http::response::Builder
|
||||
where
|
||||
http::StatusCode: std::convert::TryFrom<T>,
|
||||
<http::StatusCode as std::convert::TryFrom<T>>::Error: Into<http::Error>,
|
||||
{
|
||||
Response::builder().status(status)
|
||||
}
|
||||
|
||||
pub struct BodyStream<S> {
|
||||
inp: S,
|
||||
desc: String,
|
||||
}
|
||||
|
||||
impl<S, I> BodyStream<S>
|
||||
where
|
||||
S: Stream<Item = Result<I, Error>> + Unpin + Send + 'static,
|
||||
I: Into<Bytes> + Sized + 'static,
|
||||
{
|
||||
pub fn new(inp: S, desc: String) -> Self {
|
||||
Self { inp, desc }
|
||||
}
|
||||
|
||||
pub fn wrapped(inp: S, desc: String) -> Body {
|
||||
Body::wrap_stream(Self::new(inp, desc))
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, I> Stream for BodyStream<S>
|
||||
where
|
||||
S: Stream<Item = Result<I, Error>> + Unpin,
|
||||
I: Into<Bytes> + Sized,
|
||||
{
|
||||
type Item = Result<I, Error>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
let span1 = span!(Level::INFO, "httpret::BodyStream", desc = Empty);
|
||||
span1.record("desc", &self.desc.as_str());
|
||||
span1.in_scope(|| {
|
||||
use Poll::*;
|
||||
let t = std::panic::catch_unwind(AssertUnwindSafe(|| self.inp.poll_next_unpin(cx)));
|
||||
match t {
|
||||
Ok(r) => match r {
|
||||
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
|
||||
Ready(Some(Err(e))) => {
|
||||
error!("body stream error: {e:?}");
|
||||
Ready(Some(Err(Error::from(e))))
|
||||
}
|
||||
Ready(None) => Ready(None),
|
||||
Pending => Pending,
|
||||
},
|
||||
Err(e) => {
|
||||
error!("panic caught in httpret::BodyStream: {e:?}");
|
||||
let e = Error::with_msg(format!("panic caught in httpret::BodyStream: {e:?}"));
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ToPublicResponse {
|
||||
fn to_public_response(&self) -> Response<Body>;
|
||||
}
|
||||
|
||||
impl ToPublicResponse for Error {
|
||||
fn to_public_response(&self) -> Response<Body> {
|
||||
self.0.to_public_response()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToPublicResponse for ::err::Error {
|
||||
fn to_public_response(&self) -> Response<Body> {
|
||||
use err::Reason;
|
||||
let e = self.to_public_error();
|
||||
let status = match e.reason() {
|
||||
Some(Reason::BadRequest) => StatusCode::BAD_REQUEST,
|
||||
Some(Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
let msg = match serde_json::to_string(&e) {
|
||||
Ok(s) => s,
|
||||
Err(_) => "can not serialize error".into(),
|
||||
};
|
||||
match response(status)
|
||||
.header(http::header::ACCEPT, APP_JSON)
|
||||
.body(Body::from(msg))
|
||||
{
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
error!("can not generate http error response {e:?}");
|
||||
let mut res = Response::new(Body::default());
|
||||
*res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
|
||||
res
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BodyStreamWrap(netpod::BodyStream);
|
||||
|
||||
impl hyper::body::HttpBody for BodyStreamWrap {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = ::err::Error;
|
||||
|
||||
fn poll_data(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Self::Data, Self::Error>>> {
|
||||
self.0.inner.poll_next_unpin(cx)
|
||||
}
|
||||
|
||||
fn poll_trailers(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<Option<HeaderMap>, Self::Error>> {
|
||||
Poll::Ready(Ok(None))
|
||||
}
|
||||
}
|
||||
174
crates/httpret/src/channel_status.rs
Normal file
174
crates/httpret/src/channel_status.rs
Normal file
@@ -0,0 +1,174 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::ReqCtx;
|
||||
use futures_util::StreamExt;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use items_2::channelevents::ChannelStatusEvent;
|
||||
use items_2::channelevents::ConnStatusEvent;
|
||||
use netpod::log::*;
|
||||
use netpod::query::ChannelStateEventsQuery;
|
||||
use netpod::ChannelTypeConfigGen;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use url::Url;
|
||||
|
||||
pub struct ConnectionStatusEvents {}
|
||||
|
||||
impl ConnectionStatusEvents {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/status/connection/events" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelStateEventsQuery::from_url(&url)?;
|
||||
match self.fetch_data(&q, node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("{e}");
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::from(format!("{:?}", e.public_msg())))?)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_data(
|
||||
&self,
|
||||
q: &ChannelStateEventsQuery,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Vec<ConnStatusEvent>, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
|
||||
let _scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let chconf =
|
||||
nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config).await?;
|
||||
let _do_one_before_range = true;
|
||||
let ret = Vec::new();
|
||||
if true {
|
||||
return Err(Error::with_msg_no_trace("TODO channel_status fetch_data"));
|
||||
}
|
||||
/*let mut stream =
|
||||
scyllaconn::status::StatusStreamScylla::new(series, q.range().clone(), do_one_before_range, scy);
|
||||
while let Some(item) = stream.next().await {
|
||||
let item = item?;
|
||||
ret.push(item);
|
||||
}*/
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ChannelStatusEvents {}
|
||||
|
||||
impl ChannelStatusEvents {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/status/channel/events" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelStateEventsQuery::from_url(&url)?;
|
||||
match self.fetch_data(&q, node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("{e}");
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::from(format!("{:?}", e.public_msg())))?)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_data(
|
||||
&self,
|
||||
q: &ChannelStateEventsQuery,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Vec<ChannelStatusEvent>, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let chconf = nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config)
|
||||
.await?
|
||||
.ok_or_else(|| Error::with_msg_no_trace("channel config not found"))?;
|
||||
let do_one_before_range = true;
|
||||
match chconf {
|
||||
ChannelTypeConfigGen::Scylla(ch_conf) => {
|
||||
let mut stream = scyllaconn::status::StatusStreamScylla::new(
|
||||
ch_conf.series(),
|
||||
q.range().clone(),
|
||||
do_one_before_range,
|
||||
scy,
|
||||
);
|
||||
let mut ret = Vec::new();
|
||||
while let Some(item) = stream.next().await {
|
||||
let item = item?;
|
||||
ret.push(item);
|
||||
}
|
||||
Ok(ret)
|
||||
}
|
||||
ChannelTypeConfigGen::SfDatabuffer(k) => todo!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
974
crates/httpret/src/channelconfig.rs
Normal file
974
crates/httpret/src/channelconfig.rs
Normal file
@@ -0,0 +1,974 @@
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use crate::ToPublicResponse;
|
||||
use dbconn::create_connection;
|
||||
use futures_util::StreamExt;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::get_url_query_pairs;
|
||||
use netpod::log::*;
|
||||
use netpod::query::prebinned::PreBinnedQuery;
|
||||
use netpod::timeunits::*;
|
||||
use netpod::ChannelConfigQuery;
|
||||
use netpod::ChannelConfigResponse;
|
||||
use netpod::ChannelTypeConfigGen;
|
||||
use netpod::FromUrl;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ScalarType;
|
||||
use netpod::SfDbChannel;
|
||||
use netpod::Shape;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use query::api4::binned::BinnedQuery;
|
||||
use query::api4::events::PlainEventsQuery;
|
||||
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
|
||||
use scylla::transport::errors::NewSessionError as ScyNewSessionError;
|
||||
use scylla::transport::errors::QueryError as ScyQueryError;
|
||||
use scylla::transport::iterator::NextRowError;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
use url::Url;
|
||||
|
||||
pub async fn chconf_from_events_v1(
|
||||
q: &PlainEventsQuery,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub async fn chconf_from_prebinned(
|
||||
q: &PreBinnedQuery,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
let ret =
|
||||
nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.patch().patch_range(), ncc).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub async fn ch_conf_from_binned(
|
||||
q: &BinnedQuery,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub struct ChannelConfigHandler {}
|
||||
|
||||
impl ChannelConfigHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channel/config" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match self.channel_config(req, &node_config).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
warn!("ChannelConfigHandler::handle: got error from channel_config: {e:?}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn channel_config(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelConfigQuery::from_url(&url)?;
|
||||
let conf = nodenet::channelconfig::channel_config(q.range.clone(), q.channel.clone(), node_config).await?;
|
||||
match conf {
|
||||
Some(conf) => {
|
||||
let res: ChannelConfigResponse = conf.into();
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&res)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
None => {
|
||||
let ret = response(StatusCode::NOT_FOUND)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::empty())?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ChannelConfigsHandler {}
|
||||
|
||||
impl ChannelConfigsHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channel/configs" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match self.channel_configs(req, &node_config).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
warn!("got error from channel_config: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn channel_configs(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
info!("channel_configs");
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelConfigQuery::from_url(&url)?;
|
||||
info!("channel_configs for q {q:?}");
|
||||
let ch_confs = nodenet::channelconfig::channel_configs(q.channel, ncc).await?;
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&ch_confs)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ChannelConfigQuorumHandler {}
|
||||
|
||||
impl ChannelConfigQuorumHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channel/config/quorum" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match self.channel_config_quorum(req, &node_config).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
warn!("from channel_config_quorum: {e}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn channel_config_quorum(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
info!("channel_config_quorum");
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelConfigQuery::from_url(&url)?;
|
||||
info!("channel_config_quorum for q {q:?}");
|
||||
let ch_confs = nodenet::configquorum::find_config_basics_quorum(q.channel, q.range.into(), ncc).await?;
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&ch_confs)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
trait ErrConv<T> {
|
||||
fn err_conv(self) -> Result<T, Error>;
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, ScyQueryError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, ScyNewSessionError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, ScyFromRowError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ErrConv<T> for Result<T, NextRowError> {
|
||||
fn err_conv(self) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => Err(Error::with_msg_no_trace(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConfigsHisto {
|
||||
scalar_types: Vec<(ScalarType, Vec<(Shape, u32)>)>,
|
||||
}
|
||||
|
||||
pub struct ScyllaConfigsHisto {}
|
||||
|
||||
impl ScyllaConfigsHisto {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/scylla/configs/histo" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
let res = self
|
||||
.make_histo(&node_config.node_config.cluster.backend, node_config)
|
||||
.await?;
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_histo(&self, backend: &str, node_config: &NodeConfigCached) -> Result<ConfigsHisto, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let res = scy
|
||||
.query(
|
||||
"select scalar_type, shape_dims, series from series_by_channel where facility = ? allow filtering",
|
||||
(backend,),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
let mut stm = BTreeMap::new();
|
||||
for row in res.rows_typed_or_empty::<(i32, Vec<i32>, i64)>() {
|
||||
let (st, dims, _) = row.err_conv()?;
|
||||
let scalar_type = ScalarType::from_scylla_i32(st)?;
|
||||
let shape = Shape::from_scylla_shape_dims(&dims)?;
|
||||
if stm.get_mut(&scalar_type).is_none() {
|
||||
stm.insert(scalar_type.clone(), BTreeMap::new());
|
||||
}
|
||||
let a = stm.get_mut(&scalar_type).unwrap();
|
||||
if a.get_mut(&shape).is_none() {
|
||||
a.insert(shape.clone(), 0);
|
||||
}
|
||||
*a.get_mut(&shape).unwrap() += 1;
|
||||
}
|
||||
let mut stm: Vec<_> = stm
|
||||
.into_iter()
|
||||
.map(|(st, m2)| {
|
||||
let mut g: Vec<_> = m2.into_iter().map(|(sh, c)| (sh, c)).collect();
|
||||
g.sort_by_key(|x| !x.1);
|
||||
let n = g.len() as u32;
|
||||
(st, g, n)
|
||||
})
|
||||
.collect();
|
||||
stm.sort_unstable_by_key(|x| !x.2);
|
||||
let stm = stm.into_iter().map(|(st, a, _)| (st, a)).collect();
|
||||
let ret = ConfigsHisto { scalar_types: stm };
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ChannelsWithTypeQuery {
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
}
|
||||
|
||||
impl FromUrl for ChannelsWithTypeQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let s = pairs
|
||||
.get("scalar_type")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalar_type"))?;
|
||||
//let scalar_type = ScalarType::from_bsread_str(s)?;
|
||||
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
|
||||
let s = pairs
|
||||
.get("shape")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing shape"))?;
|
||||
let shape = Shape::from_dims_str(s)?;
|
||||
Ok(Self { scalar_type, shape })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ChannelListWithType {
|
||||
channels: Vec<SfDbChannel>,
|
||||
}
|
||||
|
||||
pub struct ScyllaChannelsWithType {}
|
||||
|
||||
impl ScyllaChannelsWithType {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/scylla/channels/with_type" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ChannelsWithTypeQuery::from_url(&url)?;
|
||||
let res = self
|
||||
.get_channels(&q, &node_config.node_config.cluster.backend, node_config)
|
||||
.await?;
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_channels(
|
||||
&self,
|
||||
q: &ChannelsWithTypeQuery,
|
||||
backend: &str,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<ChannelListWithType, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let res = scy
|
||||
.query(
|
||||
"select channel_name, series from series_by_channel where facility = ? and scalar_type = ? and shape_dims = ? allow filtering",
|
||||
(backend, q.scalar_type.to_scylla_i32(), q.shape.to_scylla_vec()),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
let mut list = Vec::new();
|
||||
for row in res.rows_typed_or_empty::<(String, i64)>() {
|
||||
let (channel_name, series) = row.err_conv()?;
|
||||
let ch = SfDbChannel::from_full(backend, Some(series as u64), channel_name);
|
||||
list.push(ch);
|
||||
}
|
||||
let ret = ChannelListWithType { channels: list };
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ScyllaChannelEventSeriesIdQuery {
|
||||
backend: String,
|
||||
#[serde(rename = "channelName")]
|
||||
name: String,
|
||||
#[serde(rename = "scalarType")]
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
#[serde(rename = "doCreate", skip_serializing_if = "bool_false")]
|
||||
do_create: bool,
|
||||
}
|
||||
|
||||
fn bool_false(x: &bool) -> bool {
|
||||
*x == false
|
||||
}
|
||||
|
||||
impl FromUrl for ScyllaChannelEventSeriesIdQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let backend = pairs
|
||||
.get("backend")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
|
||||
.into();
|
||||
let name = pairs
|
||||
.get("channelName")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing channelName"))?
|
||||
.into();
|
||||
let s = pairs
|
||||
.get("scalarType")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalarType"))?;
|
||||
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
|
||||
let s = pairs
|
||||
.get("shape")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing shape"))?;
|
||||
let shape = Shape::from_dims_str(s)?;
|
||||
let do_create = pairs.get("doCreate").map_or("false", |x| x.as_str()) == "true";
|
||||
Ok(Self {
|
||||
backend,
|
||||
name,
|
||||
scalar_type,
|
||||
shape,
|
||||
do_create,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct ScyllaChannelEventSeriesIdResponse {
|
||||
#[serde(rename = "seriesId")]
|
||||
series: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ScyllaChannelsActiveQuery {
|
||||
tsedge: u64,
|
||||
#[serde(rename = "shapeKind")]
|
||||
shape_kind: u32,
|
||||
#[serde(rename = "scalarType")]
|
||||
scalar_type: ScalarType,
|
||||
}
|
||||
|
||||
impl FromUrl for ScyllaChannelsActiveQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let s = pairs
|
||||
.get("tsedge")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing tsedge"))?;
|
||||
let tsedge: u64 = s.parse()?;
|
||||
let s = pairs
|
||||
.get("shapeKind")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing shapeKind"))?;
|
||||
let shape_kind: u32 = s.parse()?;
|
||||
let s = pairs
|
||||
.get("scalarType")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalarType"))?;
|
||||
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
|
||||
info!("parsed scalar type inp: {s:?} val: {scalar_type:?}");
|
||||
Ok(Self {
|
||||
tsedge,
|
||||
scalar_type,
|
||||
shape_kind,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ScyllaChannelsActive {}
|
||||
|
||||
impl ScyllaChannelsActive {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channels/active" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ScyllaChannelsActiveQuery::from_url(&url)?;
|
||||
let res = self.get_channels(&q, node_config).await?;
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_channels(
|
||||
&self,
|
||||
q: &ScyllaChannelsActiveQuery,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Vec<u64>, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
// Database stores tsedge/ts_msp in units of (10 sec), and we additionally map to the grid.
|
||||
let tsedge = q.tsedge / 10 / (6 * 2) * (6 * 2);
|
||||
info!(
|
||||
"ScyllaChannelsActive::get_channels tsedge {} (10s) {} (s)",
|
||||
tsedge,
|
||||
tsedge * 10
|
||||
);
|
||||
let mut ret = Vec::new();
|
||||
for part in 0..256 {
|
||||
let mut res = scy
|
||||
.query_iter(
|
||||
"select series from series_by_ts_msp where part = ? and ts_msp = ? and shape_kind = ? and scalar_type = ?",
|
||||
(part as i32, tsedge as i32, q.shape_kind as i32, q.scalar_type.to_scylla_i32()),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
while let Some(row) = res.next().await {
|
||||
let row = row.err_conv()?;
|
||||
let (series,): (i64,) = row.into_typed().err_conv()?;
|
||||
ret.push(series as u64);
|
||||
}
|
||||
}
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct IocForChannelQuery {
|
||||
#[serde(rename = "backend")]
|
||||
backend: String,
|
||||
#[serde(rename = "channelName")]
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl FromUrl for IocForChannelQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let backend = pairs
|
||||
.get("backend")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
|
||||
.into();
|
||||
let name = pairs
|
||||
.get("channelName")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing channelName"))?
|
||||
.into();
|
||||
Ok(Self { backend, name })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct IocForChannelRes {
|
||||
#[serde(rename = "iocAddr")]
|
||||
ioc_addr: String,
|
||||
}
|
||||
|
||||
pub struct IocForChannel {}
|
||||
|
||||
impl IocForChannel {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channel/ioc" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = IocForChannelQuery::from_url(&url)?;
|
||||
match self.find(&q, node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => {
|
||||
let body = Body::from(format!("{:?}", e.public_msg()));
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body)?)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn find(
|
||||
&self,
|
||||
q: &IocForChannelQuery,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Option<IocForChannelRes>, Error> {
|
||||
let dbconf = &node_config.node_config.cluster.database;
|
||||
let pg_client = create_connection(dbconf).await?;
|
||||
let rows = pg_client
|
||||
.query(
|
||||
"select addr from ioc_by_channel where facility = $1 and channel = $2",
|
||||
&[&q.backend, &q.name],
|
||||
)
|
||||
.await?;
|
||||
if let Some(row) = rows.first() {
|
||||
let ioc_addr = row.get(0);
|
||||
let ret = IocForChannelRes { ioc_addr };
|
||||
Ok(Some(ret))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct ScyllaSeriesTsMspQuery {
|
||||
#[serde(rename = "seriesId")]
|
||||
series: u64,
|
||||
}
|
||||
|
||||
impl FromUrl for ScyllaSeriesTsMspQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let s = pairs
|
||||
.get("seriesId")
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace("missing seriesId"))?;
|
||||
let series: u64 = s.parse()?;
|
||||
Ok(Self { series })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct ScyllaSeriesTsMspResponse {
|
||||
#[serde(rename = "tsMsps")]
|
||||
ts_msps: Vec<u64>,
|
||||
}
|
||||
|
||||
pub struct ScyllaSeriesTsMsp {}
|
||||
|
||||
impl ScyllaSeriesTsMsp {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/scylla/series/tsMsps" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
|
||||
let q = ScyllaSeriesTsMspQuery::from_url(&url)?;
|
||||
match self.get_ts_msps(&q, node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::from(format!("{:?}", e.public_msg())))?),
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_ts_msps(
|
||||
&self,
|
||||
q: &ScyllaSeriesTsMspQuery,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<ScyllaSeriesTsMspResponse, Error> {
|
||||
let scyco = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.scylla
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let mut ts_msps = Vec::new();
|
||||
let mut res = scy
|
||||
.query_iter("select ts_msp from ts_msp where series = ?", (q.series as i64,))
|
||||
.await
|
||||
.err_conv()?;
|
||||
while let Some(row) = res.next().await {
|
||||
let row = row.err_conv()?;
|
||||
let (ts_msp,): (i64,) = row.into_typed().err_conv()?;
|
||||
ts_msps.push(ts_msp as u64);
|
||||
}
|
||||
let ret = ScyllaSeriesTsMspResponse { ts_msps };
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct AmbigiousChannel {
|
||||
series: u64,
|
||||
name: String,
|
||||
scalar_type: ScalarType,
|
||||
shape: Shape,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct AmbigiousChannelNamesResponse {
|
||||
ambigious: Vec<AmbigiousChannel>,
|
||||
}
|
||||
|
||||
pub struct AmbigiousChannelNames {}
|
||||
|
||||
impl AmbigiousChannelNames {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/channels/ambigious" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
match self.process(node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::from(format!("{:?}", e.public_msg())))?),
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn process(&self, node_config: &NodeConfigCached) -> Result<AmbigiousChannelNamesResponse, Error> {
|
||||
let dbconf = &node_config.node_config.cluster.database;
|
||||
let pg_client = create_connection(dbconf).await?;
|
||||
let rows = pg_client
|
||||
.query(
|
||||
"select t2.series, t2.channel, t2.scalar_type, t2.shape_dims, t2.agg_kind from series_by_channel t1, series_by_channel t2 where t2.channel = t1.channel and t2.series != t1.series",
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let mut ret = AmbigiousChannelNamesResponse { ambigious: Vec::new() };
|
||||
for row in rows {
|
||||
let g = AmbigiousChannel {
|
||||
series: row.get::<_, i64>(0) as u64,
|
||||
name: row.get(1),
|
||||
scalar_type: ScalarType::from_scylla_i32(row.get(2))?,
|
||||
shape: Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(3))?,
|
||||
};
|
||||
ret.ambigious.push(g);
|
||||
}
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
struct TestData01Iter {}
|
||||
|
||||
impl Iterator for TestData01Iter {
|
||||
type Item = f64;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
struct Msps(Vec<u64>);
|
||||
struct Lsps(Vec<u64>);
|
||||
struct Pulses(Vec<u64>);
|
||||
struct ValsF64(Vec<f64>);
|
||||
|
||||
fn test_data_f64_01() -> (Msps, Lsps, Pulses, ValsF64) {
|
||||
let mut msps = Msps(Vec::new());
|
||||
let mut lsps = Lsps(Vec::new());
|
||||
let mut pulses = Pulses(Vec::new());
|
||||
let mut vals = ValsF64(Vec::new());
|
||||
let mut msp = 0;
|
||||
let mut i1 = 0;
|
||||
for i in 0..2000 {
|
||||
let ts = SEC * 1600000000 + MIN * 2 * i;
|
||||
let pulse = 10000 + i;
|
||||
if msp == 0 || i1 >= 40 {
|
||||
msp = ts / MIN * MIN;
|
||||
i1 = 0;
|
||||
}
|
||||
msps.0.push(msp);
|
||||
lsps.0.push(ts - msp);
|
||||
pulses.0.push(pulse);
|
||||
vals.0.push(pulse as f64 + 0.4 + 0.2 * (pulse as f64).sin());
|
||||
i1 += 1;
|
||||
}
|
||||
(msps, lsps, pulses, vals)
|
||||
}
|
||||
|
||||
pub struct GenerateScyllaTestData {}
|
||||
|
||||
impl GenerateScyllaTestData {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/test/generate/scylla" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept == APP_JSON || accept == ACCEPT_ALL {
|
||||
match self.process(node_config).await {
|
||||
Ok(k) => {
|
||||
let body = Body::from(serde_json::to_vec(&k)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::from(format!("{:?}", e.public_msg())))?),
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn process(&self, node_config: &NodeConfigCached) -> Result<(), Error> {
|
||||
let dbconf = &node_config.node_config.cluster.database;
|
||||
let _pg_client = create_connection(dbconf).await?;
|
||||
let scyconf = node_config.node_config.cluster.scylla.as_ref().unwrap();
|
||||
let scy = scyllaconn::create_scy_session(scyconf).await?;
|
||||
let series: u64 = 42001;
|
||||
// TODO query `ts_msp` for all MSP values und use that to delete from event table first.
|
||||
// Only later delete also from the `ts_msp` table.
|
||||
let it = scy
|
||||
.query_iter("select ts_msp from ts_msp where series = ?", (series as i64,))
|
||||
.await
|
||||
.err_conv()?;
|
||||
let mut it = it.into_typed::<(i64,)>();
|
||||
while let Some(row) = it.next().await {
|
||||
let row = row.err_conv()?;
|
||||
let values = (series as i64, row.0);
|
||||
scy.query("delete from events_scalar_f64 where series = ? and ts_msp = ?", values)
|
||||
.await
|
||||
.err_conv()?;
|
||||
}
|
||||
scy.query("delete from ts_msp where series = ?", (series as i64,))
|
||||
.await
|
||||
.err_conv()?;
|
||||
|
||||
// Generate
|
||||
let (msps, lsps, pulses, vals) = test_data_f64_01();
|
||||
let mut last = 0;
|
||||
for msp in msps.0.iter().map(|x| *x) {
|
||||
if msp != last {
|
||||
scy.query(
|
||||
"insert into ts_msp (series, ts_msp) values (?, ?)",
|
||||
(series as i64, msp as i64),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
}
|
||||
last = msp;
|
||||
}
|
||||
for (((msp, lsp), pulse), val) in msps.0.into_iter().zip(lsps.0).zip(pulses.0).zip(vals.0) {
|
||||
scy.query(
|
||||
"insert into events_scalar_f64 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?)",
|
||||
(series as i64, msp as i64, lsp as i64, pulse as i64, val),
|
||||
)
|
||||
.await
|
||||
.err_conv()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
88
crates/httpret/src/download.rs
Normal file
88
crates/httpret/src/download.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use futures_util::TryStreamExt;
|
||||
use http::{Method, StatusCode};
|
||||
use hyper::{Body, Request, Response};
|
||||
use netpod::log::*;
|
||||
use netpod::{get_url_query_pairs, DiskIoTune, FromUrl, NodeConfigCached};
|
||||
use url::Url;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DownloadQuery {
|
||||
disk_io_tune: DiskIoTune,
|
||||
}
|
||||
|
||||
impl FromUrl for DownloadQuery {
|
||||
fn from_url(url: &Url) -> Result<Self, ::err::Error> {
|
||||
let pairs = get_url_query_pairs(url);
|
||||
Self::from_pairs(&pairs)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &std::collections::BTreeMap<String, String>) -> Result<Self, err::Error> {
|
||||
let read_sys = pairs
|
||||
.get("ReadSys")
|
||||
.map(|x| x as &str)
|
||||
.unwrap_or("TokioAsyncRead")
|
||||
.into();
|
||||
let read_buffer_len = pairs
|
||||
.get("ReadBufferLen")
|
||||
.map(|x| x as &str)
|
||||
.unwrap_or("xx")
|
||||
.parse()
|
||||
.unwrap_or(1024 * 4);
|
||||
let read_queue_len = pairs
|
||||
.get("ReadQueueLen")
|
||||
.map(|x| x as &str)
|
||||
.unwrap_or("xx")
|
||||
.parse()
|
||||
.unwrap_or(8);
|
||||
let disk_io_tune = DiskIoTune {
|
||||
read_sys,
|
||||
read_buffer_len,
|
||||
read_queue_len,
|
||||
};
|
||||
let ret = Self { disk_io_tune };
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DownloadHandler {}
|
||||
|
||||
impl DownloadHandler {
|
||||
pub fn path_prefix() -> &'static str {
|
||||
"/api/4/test/download/"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path().starts_with(Self::path_prefix()) {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let p2 = &head.uri.path()[Self::path_prefix().len()..];
|
||||
let base = match &node_config.node.sf_databuffer {
|
||||
Some(k) => k.data_base_path.clone(),
|
||||
None => "/UNDEFINED".into(),
|
||||
};
|
||||
let url = url::Url::parse(&format!("http://dummy{}", head.uri))?;
|
||||
let query = DownloadQuery::from_url(&url)?;
|
||||
// TODO wrap file operation to return a better error.
|
||||
let pp = base.join(p2);
|
||||
info!("Try to open {pp:?}");
|
||||
let file = tokio::fs::OpenOptions::new().read(true).open(&pp).await?;
|
||||
let s = disk::file_content_stream(pp, file, query.disk_io_tune.clone()).map_ok(|x| x.into_buf());
|
||||
Ok(response(StatusCode::OK).body(Body::wrap_stream(s))?)
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
self.get(req, node_config).await
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
95
crates/httpret/src/err.rs
Normal file
95
crates/httpret/src/err.rs
Normal file
@@ -0,0 +1,95 @@
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Error(pub err::Error);
|
||||
|
||||
impl Error {
|
||||
pub fn with_msg<S: Into<String>>(s: S) -> Self {
|
||||
Self(err::Error::with_msg(s))
|
||||
}
|
||||
|
||||
pub fn with_msg_no_trace<S: Into<String>>(s: S) -> Self {
|
||||
Self(err::Error::with_msg_no_trace(s))
|
||||
}
|
||||
|
||||
pub fn with_public_msg<S: Into<String>>(s: S) -> Self {
|
||||
Self(err::Error::with_public_msg(s))
|
||||
}
|
||||
|
||||
pub fn with_public_msg_no_trace<S: Into<String>>(s: S) -> Self {
|
||||
Self(err::Error::with_public_msg_no_trace(s))
|
||||
}
|
||||
|
||||
pub fn msg(&self) -> &str {
|
||||
self.0.msg()
|
||||
}
|
||||
|
||||
pub fn reason(&self) -> Option<::err::Reason> {
|
||||
self.0.reason()
|
||||
}
|
||||
|
||||
pub fn public_msg(&self) -> Option<&Vec<String>> {
|
||||
self.0.public_msg()
|
||||
}
|
||||
|
||||
pub fn add_public_msg(self, msg: impl Into<String>) -> Self {
|
||||
Error(self.0.add_public_msg(msg))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Error {
|
||||
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Debug::fmt(&self.0, fmt)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt::Display::fmt(&self.0, fmt)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl From<err::Error> for Error {
|
||||
fn from(x: err::Error) -> Self {
|
||||
Self(x)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Error> for err::Error {
|
||||
fn from(x: Error) -> Self {
|
||||
x.0
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Convable {}
|
||||
|
||||
impl<T: Convable> From<T> for Error
|
||||
where
|
||||
T: ToString,
|
||||
{
|
||||
fn from(x: T) -> Self {
|
||||
Self(::err::Error::from_string(x))
|
||||
}
|
||||
}
|
||||
|
||||
impl Convable for std::net::AddrParseError {}
|
||||
impl Convable for std::string::FromUtf8Error {}
|
||||
impl Convable for fmt::Error {}
|
||||
impl Convable for std::io::Error {}
|
||||
impl Convable for std::num::ParseIntError {}
|
||||
impl Convable for dbconn::pg::Error {}
|
||||
impl Convable for tokio::task::JoinError {}
|
||||
impl Convable for tokio::time::error::Elapsed {}
|
||||
impl Convable for serde_json::Error {}
|
||||
impl Convable for chrono::ParseError {}
|
||||
impl Convable for url::ParseError {}
|
||||
impl Convable for http::uri::InvalidUri {}
|
||||
impl Convable for http::Error {}
|
||||
impl Convable for http::header::ToStrError {}
|
||||
impl Convable for hyper::Error {}
|
||||
impl Convable for std::array::TryFromSliceError {}
|
||||
impl Convable for err::anyhow::Error {}
|
||||
1
crates/httpret/src/evinfo.rs
Normal file
1
crates/httpret/src/evinfo.rs
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
293
crates/httpret/src/gather.rs
Normal file
293
crates/httpret/src/gather.rs
Normal file
@@ -0,0 +1,293 @@
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use futures_util::{select, FutureExt};
|
||||
use http::{Method, StatusCode};
|
||||
use hyper::{Body, Client, Request, Response};
|
||||
use netpod::log::*;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::{Node, NodeConfigCached};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use url::Url;
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
struct GatherFrom {
|
||||
hosts: Vec<GatherHost>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
struct GatherHost {
|
||||
host: String,
|
||||
port: u16,
|
||||
inst: String,
|
||||
}
|
||||
|
||||
async fn process_answer(res: Response<Body>) -> Result<JsonValue, Error> {
|
||||
let (pre, mut body) = res.into_parts();
|
||||
if pre.status != StatusCode::OK {
|
||||
use hyper::body::HttpBody;
|
||||
if let Some(c) = body.data().await {
|
||||
let c: bytes::Bytes = c?;
|
||||
let s1 = String::from_utf8(c.to_vec())?;
|
||||
Ok(JsonValue::String(format!(
|
||||
"status {} body {}",
|
||||
pre.status.as_str(),
|
||||
s1
|
||||
)))
|
||||
} else {
|
||||
Ok(JsonValue::String(format!("status {}", pre.status.as_str())))
|
||||
}
|
||||
} else {
|
||||
let body: hyper::Body = body;
|
||||
let body_all = hyper::body::to_bytes(body).await?;
|
||||
let val = match serde_json::from_slice(&body_all) {
|
||||
Ok(k) => k,
|
||||
Err(_e) => JsonValue::String(String::from_utf8(body_all.to_vec())?),
|
||||
};
|
||||
Ok::<_, Error>(val)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn unused_gather_json_from_hosts(req: Request<Body>, pathpre: &str) -> Result<Response<Body>, Error> {
|
||||
let (part_head, part_body) = req.into_parts();
|
||||
let bodyslice = hyper::body::to_bytes(part_body).await?;
|
||||
let gather_from: GatherFrom = serde_json::from_slice(&bodyslice)?;
|
||||
let mut spawned = vec![];
|
||||
let uri = part_head.uri;
|
||||
let path_post = &uri.path()[pathpre.len()..];
|
||||
for gh in gather_from.hosts {
|
||||
let uri = format!("http://{}:{}/{}", gh.host, gh.port, path_post);
|
||||
let req = Request::builder().method(Method::GET).uri(uri);
|
||||
let req = if gh.inst.len() > 0 {
|
||||
req.header("retrieval_instance", &gh.inst)
|
||||
} else {
|
||||
req
|
||||
};
|
||||
let req = req.header(http::header::ACCEPT, APP_JSON);
|
||||
let req = req.body(Body::empty());
|
||||
let task = tokio::spawn(async move {
|
||||
select! {
|
||||
_ = sleep(Duration::from_millis(1500)).fuse() => {
|
||||
Err(Error::with_msg("timeout"))
|
||||
}
|
||||
res = Client::new().request(req?).fuse() => Ok(process_answer(res?).await?)
|
||||
}
|
||||
});
|
||||
spawned.push((gh.clone(), task));
|
||||
}
|
||||
#[derive(Serialize)]
|
||||
struct Hres {
|
||||
gh: GatherHost,
|
||||
res: JsonValue,
|
||||
}
|
||||
#[derive(Serialize)]
|
||||
struct Jres {
|
||||
hosts: Vec<Hres>,
|
||||
}
|
||||
let mut a = vec![];
|
||||
for tr in spawned {
|
||||
let res = match tr.1.await {
|
||||
Ok(k) => match k {
|
||||
Ok(k) => k,
|
||||
Err(e) => JsonValue::String(format!("ERROR({:?})", e)),
|
||||
},
|
||||
Err(e) => JsonValue::String(format!("ERROR({:?})", e)),
|
||||
};
|
||||
a.push(Hres { gh: tr.0, res });
|
||||
}
|
||||
let res = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(serde_json::to_string(&Jres { hosts: a })?.into())?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub async fn gather_get_json(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let (head, body) = req.into_parts();
|
||||
let _bodyslice = hyper::body::to_bytes(body).await?;
|
||||
let pathpre = "/api/4/gather/";
|
||||
let pathsuf = &head.uri.path()[pathpre.len()..];
|
||||
let spawned: Vec<_> = node_config
|
||||
.node_config
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| {
|
||||
let uri = format!("http://{}:{}/api/4/{}", node.host, node.port, pathsuf);
|
||||
let req = Request::builder().method(Method::GET).uri(uri);
|
||||
let req = req.header(http::header::ACCEPT, APP_JSON);
|
||||
let req = req.body(Body::empty());
|
||||
let task = tokio::spawn(async move {
|
||||
select! {
|
||||
_ = sleep(Duration::from_millis(1500)).fuse() => {
|
||||
Err(Error::with_msg("timeout"))
|
||||
}
|
||||
res = Client::new().request(req?).fuse() => Ok(process_answer(res?).await?)
|
||||
}
|
||||
});
|
||||
(node.clone(), task)
|
||||
})
|
||||
.collect();
|
||||
#[derive(Serialize)]
|
||||
struct Hres {
|
||||
node: Node,
|
||||
res: JsonValue,
|
||||
}
|
||||
#[derive(Serialize)]
|
||||
struct Jres {
|
||||
hosts: Vec<Hres>,
|
||||
}
|
||||
let mut a = vec![];
|
||||
for (node, jh) in spawned {
|
||||
let res = match jh.await {
|
||||
Ok(k) => match k {
|
||||
Ok(k) => k,
|
||||
Err(e) => JsonValue::String(format!("ERROR({:?})", e)),
|
||||
},
|
||||
Err(e) => JsonValue::String(format!("ERROR({:?})", e)),
|
||||
};
|
||||
let v = Hres {
|
||||
node: node.clone(),
|
||||
res,
|
||||
};
|
||||
a.push(v);
|
||||
}
|
||||
let a = a;
|
||||
let res = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(serde_json::to_string(&Jres { hosts: a })?.into())?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)]
|
||||
pub struct Tag(pub String);
|
||||
|
||||
pub struct SubRes<T> {
|
||||
pub tag: String,
|
||||
pub status: StatusCode,
|
||||
pub val: T,
|
||||
}
|
||||
|
||||
pub async fn gather_get_json_generic<SM, NT, FT, OUT>(
|
||||
_method: http::Method,
|
||||
urls: Vec<Url>,
|
||||
bodies: Vec<Option<Body>>,
|
||||
tags: Vec<String>,
|
||||
nt: NT,
|
||||
ft: FT,
|
||||
// TODO use deadline instead.
|
||||
// TODO Wait a bit longer compared to remote to receive partial results.
|
||||
timeout: Duration,
|
||||
) -> Result<OUT, Error>
|
||||
where
|
||||
SM: Send + 'static,
|
||||
NT: Fn(String, Response<Body>) -> Pin<Box<dyn Future<Output = Result<SubRes<SM>, Error>> + Send>>
|
||||
+ Send
|
||||
+ Sync
|
||||
+ Copy
|
||||
+ 'static,
|
||||
FT: Fn(Vec<(Tag, Result<SubRes<SM>, Error>)>) -> Result<OUT, Error>,
|
||||
{
|
||||
// TODO remove magic constant
|
||||
let extra_timeout = Duration::from_millis(3000);
|
||||
if urls.len() != bodies.len() {
|
||||
return Err(Error::with_msg_no_trace("unequal numbers of urls and bodies"));
|
||||
}
|
||||
if urls.len() != tags.len() {
|
||||
return Err(Error::with_msg_no_trace("unequal numbers of urls and tags"));
|
||||
}
|
||||
let spawned: Vec<_> = urls
|
||||
.into_iter()
|
||||
.zip(bodies.into_iter())
|
||||
.zip(tags.into_iter())
|
||||
.map(move |((url, body), tag)| {
|
||||
info!("Try gather from {}", url);
|
||||
let url_str = url.as_str();
|
||||
let req = if body.is_some() {
|
||||
Request::builder().method(Method::POST).uri(url_str)
|
||||
} else {
|
||||
Request::builder().method(Method::GET).uri(url_str)
|
||||
};
|
||||
let req = req.header(http::header::ACCEPT, APP_JSON);
|
||||
let req = if body.is_some() {
|
||||
req.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
} else {
|
||||
req
|
||||
};
|
||||
let body = match body {
|
||||
None => Body::empty(),
|
||||
Some(body) => body,
|
||||
};
|
||||
let req = req.body(body);
|
||||
let tag2 = tag.clone();
|
||||
let jh = tokio::spawn(async move {
|
||||
select! {
|
||||
_ = sleep(timeout + extra_timeout).fuse() => {
|
||||
error!("PROXY TIMEOUT");
|
||||
Err(Error::with_msg_no_trace("timeout"))
|
||||
}
|
||||
res = {
|
||||
let client = Client::new();
|
||||
client.request(req?).fuse()
|
||||
} => {
|
||||
info!("received result in time");
|
||||
let ret = nt(tag2, res?).await?;
|
||||
info!("transformed result in time");
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
});
|
||||
(url, tag, jh)
|
||||
})
|
||||
.collect();
|
||||
let mut a = Vec::new();
|
||||
for (_url, tag, jh) in spawned {
|
||||
let res = match jh.await {
|
||||
Ok(k) => match k {
|
||||
Ok(k) => (Tag(tag), Ok(k)),
|
||||
Err(e) => {
|
||||
warn!("{e:?}");
|
||||
(Tag(tag), Err(e))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("{e:?}");
|
||||
(Tag(tag), Err(e.into()))
|
||||
}
|
||||
};
|
||||
a.push(res);
|
||||
}
|
||||
ft(a)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn try_search() {
|
||||
let fut = gather_get_json_generic(
|
||||
hyper::Method::GET,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
|tag, _res| {
|
||||
let fut = async {
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val: (),
|
||||
};
|
||||
Ok(ret)
|
||||
};
|
||||
Box::pin(fut)
|
||||
},
|
||||
|_all| Ok(String::from("DUMMY-SEARCH-TEST-RESULT-TODO")),
|
||||
Duration::from_millis(800),
|
||||
);
|
||||
let _ = fut;
|
||||
}
|
||||
}
|
||||
794
crates/httpret/src/httpret.rs
Normal file
794
crates/httpret/src/httpret.rs
Normal file
@@ -0,0 +1,794 @@
|
||||
pub mod api1;
|
||||
pub mod api4;
|
||||
pub mod bodystream;
|
||||
pub mod channel_status;
|
||||
pub mod channelconfig;
|
||||
pub mod download;
|
||||
pub mod err;
|
||||
pub mod gather;
|
||||
pub mod prometheus;
|
||||
pub mod proxy;
|
||||
pub mod pulsemap;
|
||||
pub mod settings;
|
||||
|
||||
use self::bodystream::BodyStream;
|
||||
use self::bodystream::ToPublicResponse;
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::gather::gather_get_json;
|
||||
use crate::pulsemap::UpdateTask;
|
||||
use futures_util::Future;
|
||||
use futures_util::FutureExt;
|
||||
use futures_util::StreamExt;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use hyper::server::conn::AddrStream;
|
||||
use hyper::server::Server;
|
||||
use hyper::service::make_service_fn;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::Body;
|
||||
use hyper::Request;
|
||||
use hyper::Response;
|
||||
use net::SocketAddr;
|
||||
use netpod::is_false;
|
||||
use netpod::log::*;
|
||||
use netpod::query::prebinned::PreBinnedQuery;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ProxyConfig;
|
||||
use netpod::APP_JSON;
|
||||
use netpod::APP_JSON_LINES;
|
||||
use nodenet::conn::events_service;
|
||||
use panic::AssertUnwindSafe;
|
||||
use panic::UnwindSafe;
|
||||
use pin::Pin;
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::net;
|
||||
use std::panic;
|
||||
use std::pin;
|
||||
use std::sync::atomic::AtomicPtr;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Once;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::RwLockWriteGuard;
|
||||
use std::task;
|
||||
use std::time::SystemTime;
|
||||
use task::Context;
|
||||
use task::Poll;
|
||||
|
||||
pub const PSI_DAQBUFFER_SERVICE_MARK: &'static str = "PSI-Daqbuffer-Service-Mark";
|
||||
pub const PSI_DAQBUFFER_SEEN_URL: &'static str = "PSI-Daqbuffer-Seen-Url";
|
||||
|
||||
pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
|
||||
static STATUS_BOARD_INIT: Once = Once::new();
|
||||
STATUS_BOARD_INIT.call_once(|| {
|
||||
let b = StatusBoard::new();
|
||||
let a = RwLock::new(b);
|
||||
let x = Box::new(a);
|
||||
STATUS_BOARD.store(Box::into_raw(x), Ordering::SeqCst);
|
||||
});
|
||||
if let Some(bind) = node_config.node.prometheus_api_bind {
|
||||
tokio::spawn(prometheus::host(bind));
|
||||
}
|
||||
let _update_task = if node_config.node_config.cluster.run_map_pulse_task {
|
||||
Some(UpdateTask::new(node_config.clone()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let rawjh = taskrun::spawn(events_service(node_config.clone()));
|
||||
use std::str::FromStr;
|
||||
let addr = SocketAddr::from_str(&format!("{}:{}", node_config.node.listen, node_config.node.port))?;
|
||||
let make_service = make_service_fn({
|
||||
move |conn: &AddrStream| {
|
||||
debug!("new connection from {:?}", conn.remote_addr());
|
||||
let node_config = node_config.clone();
|
||||
let addr = conn.remote_addr();
|
||||
async move {
|
||||
Ok::<_, Error>(service_fn({
|
||||
move |req| {
|
||||
// TODO send to logstash
|
||||
info!(
|
||||
"REQUEST {:?} - {:?} - {:?} - {:?}",
|
||||
addr,
|
||||
req.method(),
|
||||
req.uri(),
|
||||
req.headers()
|
||||
);
|
||||
let f = http_service(req, node_config.clone());
|
||||
Cont { f: Box::pin(f) }
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
});
|
||||
Server::bind(&addr).serve(make_service).await?;
|
||||
rawjh.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn http_service(req: Request<Body>, node_config: NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
match http_service_try(req, &node_config).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
error!("daqbuffer node http_service sees error: {}", e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Cont<F> {
|
||||
f: Pin<Box<F>>,
|
||||
}
|
||||
|
||||
impl<F, I> Future for Cont<F>
|
||||
where
|
||||
F: Future<Output = Result<I, Error>>,
|
||||
{
|
||||
type Output = <F as Future>::Output;
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
|
||||
let h = std::panic::catch_unwind(AssertUnwindSafe(|| self.f.poll_unpin(cx)));
|
||||
match h {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
error!("Cont<F> catch_unwind {:?}", e);
|
||||
match e.downcast_ref::<Error>() {
|
||||
Some(e) => {
|
||||
error!("Cont<F> catch_unwind is Error: {:?}", e);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
Poll::Ready(Err(Error::with_msg(format!("{:?}", e))))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<F> UnwindSafe for Cont<F> {}
|
||||
|
||||
pub struct ReqCtx {
|
||||
pub marks: Vec<String>,
|
||||
pub mark: String,
|
||||
}
|
||||
|
||||
impl ReqCtx {
|
||||
fn with_node<T>(req: &Request<T>, nc: &NodeConfigCached) -> Self {
|
||||
let mut marks = Vec::new();
|
||||
for (n, v) in req.headers().iter() {
|
||||
if n == PSI_DAQBUFFER_SERVICE_MARK {
|
||||
marks.push(String::from_utf8_lossy(v.as_bytes()).to_string());
|
||||
}
|
||||
}
|
||||
Self {
|
||||
marks,
|
||||
mark: format!("{}:{}", nc.node_config.name, nc.node.port),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReqCtx {
|
||||
fn with_proxy<T>(req: &Request<T>, proxy: &ProxyConfig) -> Self {
|
||||
let mut marks = Vec::new();
|
||||
for (n, v) in req.headers().iter() {
|
||||
if n == PSI_DAQBUFFER_SERVICE_MARK {
|
||||
marks.push(String::from_utf8_lossy(v.as_bytes()).to_string());
|
||||
}
|
||||
}
|
||||
Self {
|
||||
marks,
|
||||
mark: format!("{}:{}", proxy.name, proxy.port),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO remove because I want error bodies to be json.
|
||||
pub fn response_err<T>(status: StatusCode, msg: T) -> Result<Response<Body>, Error>
|
||||
where
|
||||
T: AsRef<str>,
|
||||
{
|
||||
let msg = format!(
|
||||
concat!(
|
||||
"Error:\n{}\n",
|
||||
"\nDocumentation pages API 1 and 4:",
|
||||
"\nhttps://data-api.psi.ch/api/1/documentation/",
|
||||
"\nhttps://data-api.psi.ch/api/4/documentation/",
|
||||
),
|
||||
msg.as_ref()
|
||||
);
|
||||
let ret = response(status).body(Body::from(msg))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
macro_rules! static_http {
|
||||
($path:expr, $tgt:expr, $tgtex:expr, $ctype:expr) => {
|
||||
if $path == concat!("/api/4/documentation/", $tgt) {
|
||||
let c = include_bytes!(concat!("../static/documentation/", $tgtex));
|
||||
let ret = response(StatusCode::OK)
|
||||
.header("content-type", $ctype)
|
||||
.body(Body::from(&c[..]))?;
|
||||
return Ok(ret);
|
||||
}
|
||||
};
|
||||
($path:expr, $tgt:expr, $ctype:expr) => {
|
||||
if $path == concat!("/api/4/documentation/", $tgt) {
|
||||
let c = include_bytes!(concat!("../static/documentation/", $tgt));
|
||||
let ret = response(StatusCode::OK)
|
||||
.header("content-type", $ctype)
|
||||
.body(Body::from(&c[..]))?;
|
||||
return Ok(ret);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! static_http_api1 {
|
||||
($path:expr, $tgt:expr, $tgtex:expr, $ctype:expr) => {
|
||||
if $path == concat!("/api/1/documentation/", $tgt) {
|
||||
let c = include_bytes!(concat!("../static/documentation/", $tgtex));
|
||||
let ret = response(StatusCode::OK)
|
||||
.header("content-type", $ctype)
|
||||
.body(Body::from(&c[..]))?;
|
||||
return Ok(ret);
|
||||
}
|
||||
};
|
||||
($path:expr, $tgt:expr, $ctype:expr) => {
|
||||
if $path == concat!("/api/1/documentation/", $tgt) {
|
||||
let c = include_bytes!(concat!("../static/documentation/", $tgt));
|
||||
let ret = response(StatusCode::OK)
|
||||
.header("content-type", $ctype)
|
||||
.body(Body::from(&c[..]))?;
|
||||
return Ok(ret);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
use http::HeaderValue;
|
||||
let mut urlmarks = Vec::new();
|
||||
urlmarks.push(format!("{}:{}", req.method(), req.uri()));
|
||||
for (k, v) in req.headers() {
|
||||
if k == PSI_DAQBUFFER_SEEN_URL {
|
||||
let s = String::from_utf8_lossy(v.as_bytes());
|
||||
urlmarks.push(s.into());
|
||||
}
|
||||
}
|
||||
let ctx = ReqCtx::with_node(&req, node_config);
|
||||
let mut res = http_service_inner(req, &ctx, node_config).await?;
|
||||
let hm = res.headers_mut();
|
||||
hm.append("Access-Control-Allow-Origin", "*".parse().unwrap());
|
||||
hm.append("Access-Control-Allow-Headers", "*".parse().unwrap());
|
||||
for m in &ctx.marks {
|
||||
hm.append(PSI_DAQBUFFER_SERVICE_MARK, m.parse().unwrap());
|
||||
}
|
||||
hm.append(PSI_DAQBUFFER_SERVICE_MARK, ctx.mark.parse().unwrap());
|
||||
for s in urlmarks {
|
||||
let v = HeaderValue::from_str(&s).unwrap_or_else(|_| HeaderValue::from_static("invalid"));
|
||||
hm.append(PSI_DAQBUFFER_SEEN_URL, v);
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
async fn http_service_inner(
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let uri = req.uri().clone();
|
||||
let path = uri.path();
|
||||
if path == "/api/4/private/version" {
|
||||
if req.method() == Method::GET {
|
||||
let ver_maj: u32 = std::env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap_or(0);
|
||||
let ver_min: u32 = std::env!("CARGO_PKG_VERSION_MINOR").parse().unwrap_or(0);
|
||||
let ver_pat: u32 = std::env!("CARGO_PKG_VERSION_PATCH").parse().unwrap_or(0);
|
||||
let ret = serde_json::json!({
|
||||
"daqbuf_version": {
|
||||
"major": ver_maj,
|
||||
"minor": ver_min,
|
||||
"patch": ver_pat,
|
||||
},
|
||||
});
|
||||
Ok(response(StatusCode::OK).body(Body::from(serde_json::to_vec(&ret)?))?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path.starts_with("/api/4/private/logtest/") {
|
||||
if req.method() == Method::GET {
|
||||
if path.ends_with("/trace") {
|
||||
trace!("test trace log output");
|
||||
} else if path.ends_with("/debug") {
|
||||
debug!("test debug log output");
|
||||
} else if path.ends_with("/info") {
|
||||
info!("test info log output");
|
||||
} else if path.ends_with("/warn") {
|
||||
warn!("test warn log output");
|
||||
} else if path.ends_with("/error") {
|
||||
error!("test error log output");
|
||||
} else {
|
||||
error!("test unknown log output");
|
||||
}
|
||||
Ok(response(StatusCode::OK).body(Body::empty())?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if let Some(h) = api4::status::StatusNodesRecursive::handler(&req) {
|
||||
h.handle(req, ctx, &node_config).await
|
||||
} else if let Some(h) = StatusBoardAllHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = api4::search::ChannelSearchHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = api4::binned::BinnedHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ChannelConfigQuorumHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ChannelConfigsHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ChannelConfigHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ScyllaChannelsWithType::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::IocForChannel::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ScyllaChannelsActive::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::ScyllaSeriesTsMsp::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channelconfig::AmbigiousChannelNames::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = api4::events::EventsHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = channel_status::ConnectionStatusEvents::handler(&req) {
|
||||
h.handle(req, ctx, &node_config).await
|
||||
} else if let Some(h) = channel_status::ChannelStatusEvents::handler(&req) {
|
||||
h.handle(req, ctx, &node_config).await
|
||||
} else if path == "/api/4/prebinned" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(prebinned(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path == "/api/4/random/channel" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(random_channel(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path.starts_with("/api/4/gather/") {
|
||||
if req.method() == Method::GET {
|
||||
Ok(gather_get_json(req, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path == "/api/4/clear_cache" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(clear_cache_all(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path == "/api/4/update_db_with_channel_names" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(update_db_with_channel_names(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path == "/api/4/update_db_with_all_channel_configs" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(update_db_with_all_channel_configs(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path == "/api/4/update_search_cache" {
|
||||
if req.method() == Method::GET {
|
||||
Ok(update_search_cache(req, ctx, &node_config).await?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if let Some(h) = download::DownloadHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = settings::SettingsThreadsMaxHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = api1::Api1EventsBinaryHandler::handler(&req) {
|
||||
h.handle(req, ctx, &node_config).await
|
||||
} else if let Some(h) = pulsemap::MapPulseScyllaHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::IndexFullHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::MarkClosedHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::MapPulseLocalHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::MapPulseHistoHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::MapPulseHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::Api4MapPulse2HttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = pulsemap::Api4MapPulseHttpFunction::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if let Some(h) = api1::RequestStatusHandler::handler(&req) {
|
||||
h.handle(req, &node_config).await
|
||||
} else if path.starts_with("/api/1/documentation/") {
|
||||
if req.method() == Method::GET {
|
||||
api_1_docs(path)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path.starts_with("/api/4/documentation/") {
|
||||
if req.method() == Method::GET {
|
||||
api_4_docs(path)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
use std::fmt::Write;
|
||||
let mut body = String::new();
|
||||
let out = &mut body;
|
||||
write!(out, "<pre>\n")?;
|
||||
write!(out, "METHOD {:?}<br>\n", req.method())?;
|
||||
write!(out, "URI {:?}<br>\n", req.uri())?;
|
||||
write!(out, "HOST {:?}<br>\n", req.uri().host())?;
|
||||
write!(out, "PORT {:?}<br>\n", req.uri().port())?;
|
||||
write!(out, "PATH {:?}<br>\n", req.uri().path())?;
|
||||
write!(out, "QUERY {:?}<br>\n", req.uri().query())?;
|
||||
for (hn, hv) in req.headers() {
|
||||
write!(out, "HEADER {hn:?}: {hv:?}<br>\n")?;
|
||||
}
|
||||
write!(out, "</pre>\n")?;
|
||||
Ok(response(StatusCode::NOT_FOUND).body(Body::from(body))?)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn api_4_docs(path: &str) -> Result<Response<Body>, Error> {
|
||||
static_http!(path, "", "api4.html", "text/html");
|
||||
static_http!(path, "style.css", "text/css");
|
||||
static_http!(path, "script.js", "text/javascript");
|
||||
static_http!(path, "status-main.html", "text/html");
|
||||
Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
|
||||
}
|
||||
|
||||
pub fn api_1_docs(path: &str) -> Result<Response<Body>, Error> {
|
||||
static_http_api1!(path, "", "api1.html", "text/html");
|
||||
static_http_api1!(path, "style.css", "text/css");
|
||||
static_http_api1!(path, "script.js", "text/javascript");
|
||||
Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
|
||||
}
|
||||
|
||||
pub struct StatusBoardAllHandler {}
|
||||
|
||||
impl StatusBoardAllHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/status/board/all" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, _req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
use std::ops::Deref;
|
||||
let sb = status_board().unwrap();
|
||||
let buf = serde_json::to_vec(sb.deref()).unwrap();
|
||||
let res = response(StatusCode::OK).body(Body::from(buf))?;
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
async fn prebinned(req: Request<Body>, ctx: &ReqCtx, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
match prebinned_inner(req, ctx, node_config).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(e) => {
|
||||
error!("fn prebinned: {e:?}");
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::from(e.msg().to_string()))?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn prebinned_inner(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
_node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let url: url::Url = format!("dummy://{}", head.uri).parse()?;
|
||||
let query = PreBinnedQuery::from_url(&url)?;
|
||||
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &query.patch().span_desc());
|
||||
span1.in_scope(|| {
|
||||
debug!("begin");
|
||||
});
|
||||
error!("TODO hhtpret prebinned_inner");
|
||||
//let fut = disk::binned::prebinned::pre_binned_bytes_for_http(node_config, &query).instrument(span1);
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn random_channel(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (_head, _body) = req.into_parts();
|
||||
let ret = dbconn::random_channel(node_config).await?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(ret))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn clear_cache_all(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let dry = match head.uri.query() {
|
||||
Some(q) => q.contains("dry"),
|
||||
None => false,
|
||||
};
|
||||
let res = disk::cache::clear_cache_all(node_config, dry).await?;
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&res)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn update_db_with_channel_names(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
info!("httpret::update_db_with_channel_names");
|
||||
let (head, _body) = req.into_parts();
|
||||
let _dry = match head.uri.query() {
|
||||
Some(q) => q.contains("dry"),
|
||||
None => false,
|
||||
};
|
||||
let res =
|
||||
dbconn::scan::update_db_with_channel_names(node_config.clone(), &node_config.node_config.cluster.database)
|
||||
.await;
|
||||
match res {
|
||||
Ok(res) => {
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
|
||||
.body(Body::wrap_stream(res.map(|k| match serde_json::to_string(&k) {
|
||||
Ok(mut item) => {
|
||||
item.push('\n');
|
||||
Ok(item)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
})))?;
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
let p = serde_json::to_string(&e)?;
|
||||
let res = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
|
||||
.body(Body::from(p))?;
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
async fn update_db_with_channel_names_3(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let _dry = match head.uri.query() {
|
||||
Some(q) => q.contains("dry"),
|
||||
None => false,
|
||||
};
|
||||
let res = dbconn::scan::update_db_with_channel_names_3(node_config);
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
|
||||
.body(Body::wrap_stream(res.map(|k| match serde_json::to_string(&k) {
|
||||
Ok(mut item) => {
|
||||
item.push('\n');
|
||||
Ok(item)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
})))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn update_db_with_all_channel_configs(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let _dry = match head.uri.query() {
|
||||
Some(q) => q.contains("dry"),
|
||||
None => false,
|
||||
};
|
||||
let res = dbconn::scan::update_db_with_all_channel_configs(node_config.clone()).await?;
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
|
||||
.body(Body::wrap_stream(res.map(|k| match serde_json::to_string(&k) {
|
||||
Ok(mut item) => {
|
||||
item.push('\n');
|
||||
Ok(item)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
})))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn update_search_cache(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let _dry = match head.uri.query() {
|
||||
Some(q) => q.contains("dry"),
|
||||
None => false,
|
||||
};
|
||||
let res = dbconn::scan::update_search_cache(node_config).await?;
|
||||
let ret = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&res)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct StatusBoardEntry {
|
||||
#[allow(unused)]
|
||||
#[serde(serialize_with = "instant_serde::ser")]
|
||||
ts_created: SystemTime,
|
||||
#[serde(serialize_with = "instant_serde::ser")]
|
||||
ts_updated: SystemTime,
|
||||
#[serde(skip_serializing_if = "is_false")]
|
||||
is_error: bool,
|
||||
#[serde(skip_serializing_if = "is_false")]
|
||||
is_ok: bool,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
errors: Vec<Error>,
|
||||
}
|
||||
|
||||
mod instant_serde {
|
||||
use super::*;
|
||||
use netpod::DATETIME_FMT_3MS;
|
||||
use serde::Serializer;
|
||||
pub fn ser<S: Serializer>(x: &SystemTime, ser: S) -> Result<S::Ok, S::Error> {
|
||||
use chrono::LocalResult;
|
||||
let dur = x.duration_since(std::time::UNIX_EPOCH).unwrap();
|
||||
let res = chrono::TimeZone::timestamp_opt(&chrono::Utc, dur.as_secs() as i64, dur.subsec_nanos());
|
||||
match res {
|
||||
LocalResult::None => Err(serde::ser::Error::custom(format!("Bad local instant conversion"))),
|
||||
LocalResult::Single(dt) => {
|
||||
let s = dt.format(DATETIME_FMT_3MS).to_string();
|
||||
ser.serialize_str(&s)
|
||||
}
|
||||
LocalResult::Ambiguous(dt, _dt2) => {
|
||||
let s = dt.format(DATETIME_FMT_3MS).to_string();
|
||||
ser.serialize_str(&s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StatusBoardEntry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
ts_created: SystemTime::now(),
|
||||
ts_updated: SystemTime::now(),
|
||||
is_error: false,
|
||||
is_ok: false,
|
||||
errors: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct StatusBoard {
|
||||
entries: BTreeMap<String, StatusBoardEntry>,
|
||||
}
|
||||
|
||||
impl StatusBoard {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_status_id(&mut self) -> String {
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
self.clean();
|
||||
let mut f = File::open("/dev/urandom").unwrap();
|
||||
let mut buf = [0; 8];
|
||||
f.read_exact(&mut buf).unwrap();
|
||||
let n = u64::from_le_bytes(buf);
|
||||
let s = format!("{:016x}", n);
|
||||
info!("new_status_id {s}");
|
||||
self.entries.insert(s.clone(), StatusBoardEntry::new());
|
||||
s
|
||||
}
|
||||
|
||||
pub fn clean(&mut self) {
|
||||
if self.entries.len() > 15000 {
|
||||
let mut tss: Vec<_> = self.entries.values().map(|e| e.ts_updated).collect();
|
||||
tss.sort_unstable();
|
||||
let tss = tss;
|
||||
let tsm = tss[tss.len() / 3];
|
||||
let a = std::mem::replace(&mut self.entries, BTreeMap::new());
|
||||
self.entries = a.into_iter().filter(|(_k, v)| v.ts_updated >= tsm).collect();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark_alive(&mut self, status_id: &str) {
|
||||
match self.entries.get_mut(status_id) {
|
||||
Some(e) => {
|
||||
e.ts_updated = SystemTime::now();
|
||||
}
|
||||
None => {
|
||||
error!("can not find status id {}", status_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark_ok(&mut self, status_id: &str) {
|
||||
match self.entries.get_mut(status_id) {
|
||||
Some(e) => {
|
||||
e.ts_updated = SystemTime::now();
|
||||
if !e.is_error {
|
||||
e.is_ok = true;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
error!("can not find status id {}", status_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_error(&mut self, status_id: &str, error: Error) {
|
||||
match self.entries.get_mut(status_id) {
|
||||
Some(e) => {
|
||||
e.ts_updated = SystemTime::now();
|
||||
e.is_error = true;
|
||||
e.is_ok = false;
|
||||
e.errors.push(error);
|
||||
}
|
||||
None => {
|
||||
error!("can not find status id {}", status_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn status_as_json(&self, status_id: &str) -> String {
|
||||
#[derive(Serialize)]
|
||||
struct StatJs {
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
errors: Vec<::err::PublicError>,
|
||||
}
|
||||
match self.entries.get(status_id) {
|
||||
Some(e) => {
|
||||
if e.is_ok {
|
||||
let js = StatJs { errors: Vec::new() };
|
||||
return serde_json::to_string(&js).unwrap();
|
||||
} else if e.is_error {
|
||||
let errors = e.errors.iter().map(|e| (&e.0).into()).collect();
|
||||
let js = StatJs { errors };
|
||||
return serde_json::to_string(&js).unwrap();
|
||||
} else {
|
||||
warn!("requestStatus for unfinished {status_id}");
|
||||
let js = StatJs { errors: Vec::new() };
|
||||
return serde_json::to_string(&js).unwrap();
|
||||
}
|
||||
}
|
||||
None => {
|
||||
error!("can not find status id {}", status_id);
|
||||
let e = ::err::Error::with_public_msg_no_trace(format!("Request status ID unknown {status_id}"));
|
||||
let js = StatJs { errors: vec![e.into()] };
|
||||
return serde_json::to_string(&js).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static STATUS_BOARD: AtomicPtr<RwLock<StatusBoard>> = AtomicPtr::new(std::ptr::null_mut());
|
||||
|
||||
pub fn status_board() -> Result<RwLockWriteGuard<'static, StatusBoard>, Error> {
|
||||
let x = unsafe { &*STATUS_BOARD.load(Ordering::SeqCst) }.write();
|
||||
match x {
|
||||
Ok(x) => Ok(x),
|
||||
Err(e) => Err(Error::with_msg(format!("{e:?}"))),
|
||||
}
|
||||
}
|
||||
432
crates/httpret/src/prometheus.rs
Normal file
432
crates/httpret/src/prometheus.rs
Normal file
@@ -0,0 +1,432 @@
|
||||
use crate::err::Error;
|
||||
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode};
|
||||
use hyper::server::conn::AddrStream;
|
||||
use hyper::service::{make_service_fn, service_fn};
|
||||
use hyper::{Body, Server};
|
||||
use netpod::log::*;
|
||||
use netpod::{ACCEPT_ALL, APP_JSON};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Once;
|
||||
|
||||
fn response<T>(status: T) -> http::response::Builder
|
||||
where
|
||||
http::StatusCode: std::convert::TryFrom<T>,
|
||||
<http::StatusCode as std::convert::TryFrom<T>>::Error: Into<http::Error>,
|
||||
{
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("Access-Control-Allow-Origin", "*")
|
||||
.header("Access-Control-Allow-Headers", "*")
|
||||
}
|
||||
|
||||
fn accepts_json(headers: &HeaderMap<HeaderValue>) -> bool {
|
||||
match headers.get(http::header::ACCEPT) {
|
||||
Some(h) => match h.to_str() {
|
||||
Ok(hv) => {
|
||||
if hv.contains(APP_JSON) {
|
||||
true
|
||||
} else if hv.contains(ACCEPT_ALL) {
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(_) => false,
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StatusBuildInfoHandler {}
|
||||
|
||||
impl StatusBuildInfoHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/status/buildinfo" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
info!("{} for {:?}", std::any::type_name::<Self>(), req);
|
||||
if req.method() == Method::GET {
|
||||
if accepts_json(req.headers()) {
|
||||
if true {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": {
|
||||
"version": "2.37",
|
||||
"revision": "daqingest",
|
||||
"branch": "dev",
|
||||
"buildUser": "dominik.werder",
|
||||
"buildDate": "2022-07-21",
|
||||
"goVersion": "nogo"
|
||||
}
|
||||
});
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("error")))?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SeriesHandler {}
|
||||
|
||||
impl SeriesHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/series" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
info!("{} for {:?}", std::any::type_name::<Self>(), req);
|
||||
if req.method() == Method::GET || req.method() == Method::POST {
|
||||
if accepts_json(req.headers()) {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": [
|
||||
{
|
||||
"__name__": "series1",
|
||||
//"job": "daqingest",
|
||||
//"instance": "node1"
|
||||
}
|
||||
]
|
||||
});
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
info!("series handler with {:?}", req);
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MetadataHandler {}
|
||||
|
||||
impl MetadataHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/metadata" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
info!("{} for {:?}", std::any::type_name::<Self>(), req);
|
||||
if req.method() == Method::GET {
|
||||
if accepts_json(req.headers()) {
|
||||
if true {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": {}
|
||||
});
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("error")))?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
info!("metadata handler with {:?}", req);
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LabelsHandler {}
|
||||
|
||||
impl LabelsHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/labels" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let self_name = std::any::type_name::<Self>();
|
||||
info!("{} for {:?}", self_name, req);
|
||||
if req.method() == Method::GET || req.method() == Method::POST {
|
||||
if accepts_json(req.headers()) {
|
||||
if true {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": ["__name__"]
|
||||
});
|
||||
info!("return labels {:?}", res);
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("error")))?)
|
||||
}
|
||||
} else {
|
||||
warn!("{} BAD_REQUEST {:?}", self_name, req);
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
warn!("{} METHOD_NOT_ALLOWED {:?}", self_name, req);
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LabelValuesHandler {
|
||||
label: String,
|
||||
}
|
||||
|
||||
impl LabelValuesHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
use regex::Regex;
|
||||
static mut RE1: Option<Regex> = None;
|
||||
static RE1_INIT: Once = Once::new();
|
||||
let re1 = unsafe {
|
||||
RE1_INIT.call_once(|| {
|
||||
RE1 = Some(Regex::new(r#"/api/v1/label/([-:._a-zA-Z0-9]+)/values"#).unwrap());
|
||||
});
|
||||
RE1.as_mut().unwrap_unchecked()
|
||||
};
|
||||
if let Some(caps) = re1.captures(req.uri().path()) {
|
||||
if let Some(label) = caps.get(1) {
|
||||
Some(LabelValuesHandler {
|
||||
label: label.as_str().into(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let self_name = std::any::type_name::<Self>();
|
||||
info!("{} for {:?}", self_name, req);
|
||||
info!("LABEL {:?}", self.label);
|
||||
if req.method() == Method::GET || req.method() == Method::POST {
|
||||
if accepts_json(req.headers()) {
|
||||
if self.label == "__name__" {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": ["series1", "series2"]
|
||||
});
|
||||
info!("label values {:?}", res);
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
} else {
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": []
|
||||
});
|
||||
warn!("return empty label values");
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
} else {
|
||||
warn!("{} bad accept", self_name);
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
warn!("{} label value handler with {:?}", self_name, req);
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct QueryHandler {}
|
||||
|
||||
impl QueryHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/query" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
info!("{} for {:?}", std::any::type_name::<Self>(), req);
|
||||
let url = url::Url::parse(&format!("dummy://{}", &req.uri()));
|
||||
info!("/api/v1/query parsed url: {:?}", url);
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let body_str = String::from_utf8_lossy(&body);
|
||||
info!("/api/v1/query body_str: {:?}", body_str);
|
||||
let formurl = url::Url::parse(&format!("dummy:///?{}", body_str));
|
||||
info!("/api/v1/query formurl: {:?}", formurl);
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "scalar",
|
||||
"result": [40, "2"]
|
||||
}
|
||||
});
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct QueryRangeHandler {}
|
||||
|
||||
impl QueryRangeHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/v1/query_range" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
info!("{} for {:?}", std::any::type_name::<Self>(), req);
|
||||
let url = url::Url::parse(&format!("dummy://{}", &req.uri()));
|
||||
info!("/api/v1/query_range parsed url: {:?}", url);
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let body_str = String::from_utf8_lossy(&body);
|
||||
info!("/api/v1/query_range body_str: {:?}", body_str);
|
||||
let formurl = url::Url::parse(&format!("dummy:///?{}", body_str));
|
||||
info!("/api/v1/query_range formurl: {:?}", formurl);
|
||||
match formurl {
|
||||
Ok(formurl) => {
|
||||
for (k, v) in formurl.query_pairs() {
|
||||
info!("KEY {} VALUE {}", k, v);
|
||||
}
|
||||
let qps: BTreeMap<_, _> = formurl.query_pairs().collect();
|
||||
if let (Some(ts1), Some(ts2)) = (qps.get("start"), qps.get("end")) {
|
||||
let ts1: f64 = ts1.parse().unwrap_or(1.);
|
||||
let ts2: f64 = ts2.parse().unwrap_or(2.);
|
||||
let ts1 = if !ts1.is_normal() { 1. } else { ts1 };
|
||||
let ts2 = if !ts2.is_normal() { 2. } else { ts2 };
|
||||
let ts1 = ts1.min(3e9);
|
||||
let ts2 = ts2.min(3e9);
|
||||
let ts2 = if ts2 <= ts1 { ts1 + 1. } else { ts2 };
|
||||
let dt = (ts2 - ts1) / 20.;
|
||||
info!("ts1 {} ts2 {} dt {}", ts1, ts2, dt);
|
||||
let mut ts = ts1;
|
||||
let mut vals = Vec::new();
|
||||
while ts < ts2 {
|
||||
ts += dt;
|
||||
vals.push((ts, 1.2f32));
|
||||
}
|
||||
let v: Vec<_> = vals
|
||||
.into_iter()
|
||||
.map(|(k, v)| json!([json!(k), Value::String(v.to_string())]))
|
||||
.collect();
|
||||
let res = serde_json::json!({
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "matrix",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "series1",
|
||||
},
|
||||
"values": v
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
info!("return {:?}", res);
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
return Ok(response(StatusCode::OK).body(body)?);
|
||||
} else {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("can not parse {e:?}");
|
||||
}
|
||||
}
|
||||
info!("query range returning default empty");
|
||||
let res = json!({
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "matrix",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "series1",
|
||||
},
|
||||
"values": [
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
let body = Body::from(serde_json::to_vec(&res)?);
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn http_service_inner(req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
if let Some(h) = StatusBuildInfoHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = SeriesHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = MetadataHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = LabelsHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = LabelValuesHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = QueryHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else if let Some(h) = QueryRangeHandler::handler(&req) {
|
||||
h.handle(req).await
|
||||
} else {
|
||||
warn!("no handler found for {:?}", req);
|
||||
Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn http_service(req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
match http_service_inner(req).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
error!("daqbuffer node http_service sees error: {}", e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn host(bind: SocketAddr) -> Result<(), Error> {
|
||||
let make_service = make_service_fn({
|
||||
move |conn: &AddrStream| {
|
||||
let addr = conn.remote_addr();
|
||||
async move {
|
||||
Ok::<_, Error>(service_fn({
|
||||
move |req| {
|
||||
info!(
|
||||
"REQUEST {:?} - {:?} - {:?} - {:?}",
|
||||
addr,
|
||||
req.method(),
|
||||
req.uri(),
|
||||
req.headers()
|
||||
);
|
||||
let f = http_service(req);
|
||||
Box::pin(f)
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
});
|
||||
Server::bind(&bind).serve(make_service).await?;
|
||||
Ok(())
|
||||
}
|
||||
579
crates/httpret/src/proxy.rs
Normal file
579
crates/httpret/src/proxy.rs
Normal file
@@ -0,0 +1,579 @@
|
||||
pub mod api1;
|
||||
pub mod api4;
|
||||
|
||||
use crate::api1::channel_search_configs_v1;
|
||||
use crate::api1::channel_search_list_v1;
|
||||
use crate::api1::gather_json_2_v1;
|
||||
use crate::api_1_docs;
|
||||
use crate::api_4_docs;
|
||||
use crate::err::Error;
|
||||
use crate::gather::gather_get_json_generic;
|
||||
use crate::gather::SubRes;
|
||||
use crate::pulsemap::MapPulseQuery;
|
||||
use crate::response;
|
||||
use crate::response_err;
|
||||
use crate::Cont;
|
||||
use crate::ReqCtx;
|
||||
use crate::PSI_DAQBUFFER_SERVICE_MARK;
|
||||
use futures_util::pin_mut;
|
||||
use futures_util::Stream;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use hyper::service::make_service_fn;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::Body;
|
||||
use hyper::Request;
|
||||
use hyper::Response;
|
||||
use hyper::Server;
|
||||
use itertools::Itertools;
|
||||
use netpod::log::*;
|
||||
use netpod::query::ChannelStateEventsQuery;
|
||||
use netpod::AppendToUrl;
|
||||
use netpod::ChannelConfigQuery;
|
||||
use netpod::ChannelSearchQuery;
|
||||
use netpod::ChannelSearchResult;
|
||||
use netpod::ChannelSearchSingleResult;
|
||||
use netpod::FromUrl;
|
||||
use netpod::HasBackend;
|
||||
use netpod::HasTimeout;
|
||||
use netpod::ProxyConfig;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use query::api4::binned::BinnedQuery;
|
||||
use query::api4::events::PlainEventsQuery;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::future::Future;
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::io::ReadBuf;
|
||||
use url::Url;
|
||||
|
||||
const DISTRI_PRE: &str = "/distri/";
|
||||
|
||||
pub async fn proxy(proxy_config: ProxyConfig) -> Result<(), Error> {
|
||||
use std::str::FromStr;
|
||||
let addr = SocketAddr::from_str(&format!("{}:{}", proxy_config.listen, proxy_config.port))?;
|
||||
let make_service = make_service_fn({
|
||||
move |_conn| {
|
||||
let proxy_config = proxy_config.clone();
|
||||
async move {
|
||||
Ok::<_, Error>(service_fn({
|
||||
move |req| {
|
||||
// TODO send to logstash
|
||||
info!(
|
||||
"REQUEST {:?} - {:?} - {:?} - {:?}",
|
||||
addr,
|
||||
req.method(),
|
||||
req.uri(),
|
||||
req.headers()
|
||||
);
|
||||
let f = proxy_http_service(req, proxy_config.clone());
|
||||
Cont { f: Box::pin(f) }
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
});
|
||||
Server::bind(&addr).serve(make_service).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn proxy_http_service(req: Request<Body>, proxy_config: ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
match proxy_http_service_try(req, &proxy_config).await {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
error!("data_api_proxy sees error: {:?}", e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn proxy_http_service_try(req: Request<Body>, proxy_config: &ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
let ctx = ReqCtx::with_proxy(&req, proxy_config);
|
||||
let mut res = proxy_http_service_inner(req, &ctx, proxy_config).await?;
|
||||
let hm = res.headers_mut();
|
||||
hm.insert("Access-Control-Allow-Origin", "*".parse().unwrap());
|
||||
hm.insert("Access-Control-Allow-Headers", "*".parse().unwrap());
|
||||
for m in &ctx.marks {
|
||||
hm.append(PSI_DAQBUFFER_SERVICE_MARK, m.parse().unwrap());
|
||||
}
|
||||
hm.append(PSI_DAQBUFFER_SERVICE_MARK, ctx.mark.parse().unwrap());
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
async fn proxy_http_service_inner(
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
proxy_config: &ProxyConfig,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let uri = req.uri().clone();
|
||||
let path = uri.path();
|
||||
if path == "/api/1/channels" {
|
||||
Ok(channel_search_list_v1(req, proxy_config).await?)
|
||||
} else if path == "/api/1/channels/config" {
|
||||
Ok(channel_search_configs_v1(req, proxy_config).await?)
|
||||
} else if path.starts_with("/api/1/gather/") {
|
||||
Ok(gather_json_2_v1(req, "/api/1/gather/", proxy_config).await?)
|
||||
} else if path == "/api/4/private/version" {
|
||||
if req.method() == Method::GET {
|
||||
let ver_maj: u32 = std::env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap_or(0);
|
||||
let ver_min: u32 = std::env!("CARGO_PKG_VERSION_MINOR").parse().unwrap_or(0);
|
||||
let ver_pat: u32 = std::env!("CARGO_PKG_VERSION_PATCH").parse().unwrap_or(0);
|
||||
let ret = serde_json::json!({
|
||||
"daqbuf_version": {
|
||||
"major": ver_maj,
|
||||
"minor": ver_min,
|
||||
"patch": ver_pat,
|
||||
},
|
||||
});
|
||||
Ok(response(StatusCode::OK).body(Body::from(serde_json::to_vec(&ret)?))?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if let Some(h) = api4::StatusNodesRecursive::handler(&req) {
|
||||
h.handle(req, ctx, &proxy_config).await
|
||||
} else if path == "/api/4/backends" {
|
||||
Ok(backends(req, proxy_config).await?)
|
||||
} else if let Some(h) = api4::ChannelSearchAggHandler::handler(&req) {
|
||||
h.handle(req, &proxy_config).await
|
||||
} else if path == "/api/4/events" {
|
||||
Ok(proxy_single_backend_query::<PlainEventsQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path == "/api/4/status/connection/events" {
|
||||
Ok(proxy_single_backend_query::<ChannelStateEventsQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path == "/api/4/status/channel/events" {
|
||||
Ok(proxy_single_backend_query::<ChannelStateEventsQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path.starts_with("/api/4/map/pulse-v2/") {
|
||||
Ok(proxy_single_backend_query::<MapPulseQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path.starts_with("/api/4/map/pulse/") {
|
||||
Ok(proxy_single_backend_query::<MapPulseQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path == "/api/4/binned" {
|
||||
Ok(proxy_single_backend_query::<BinnedQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path == "/api/4/channel/config" {
|
||||
Ok(proxy_single_backend_query::<ChannelConfigQuery>(req, ctx, proxy_config).await?)
|
||||
} else if path.starts_with("/api/1/documentation/") {
|
||||
if req.method() == Method::GET {
|
||||
api_1_docs(path)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path.starts_with("/api/4/documentation/") {
|
||||
if req.method() == Method::GET {
|
||||
api_4_docs(path)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
} else if path.starts_with("/api/4/test/http/204") {
|
||||
Ok(response(StatusCode::NO_CONTENT).body(Body::from("No Content"))?)
|
||||
} else if path.starts_with("/api/4/test/http/400") {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::from("Bad Request"))?)
|
||||
} else if path.starts_with("/api/4/test/http/405") {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::from("Method Not Allowed"))?)
|
||||
} else if path.starts_with("/api/4/test/http/406") {
|
||||
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::from("Not Acceptable"))?)
|
||||
} else if path.starts_with("/api/4/test/log/error") {
|
||||
error!("{path}");
|
||||
Ok(response(StatusCode::OK).body(Body::empty())?)
|
||||
} else if path.starts_with("/api/4/test/log/warn") {
|
||||
warn!("{path}");
|
||||
Ok(response(StatusCode::OK).body(Body::empty())?)
|
||||
} else if path.starts_with("/api/4/test/log/info") {
|
||||
info!("{path}");
|
||||
Ok(response(StatusCode::OK).body(Body::empty())?)
|
||||
} else if path.starts_with("/api/4/test/log/debug") {
|
||||
debug!("{path}");
|
||||
Ok(response(StatusCode::OK).body(Body::empty())?)
|
||||
} else if let Some(h) = api1::PythonDataApi1Query::handler(&req) {
|
||||
h.handle(req, ctx, proxy_config).await
|
||||
} else if let Some(h) = api1::reqstatus::RequestStatusHandler::handler(&req) {
|
||||
h.handle(req, proxy_config).await
|
||||
} else if path.starts_with(DISTRI_PRE) {
|
||||
proxy_distribute_v2(req).await
|
||||
} else {
|
||||
use std::fmt::Write;
|
||||
let mut body = String::new();
|
||||
let out = &mut body;
|
||||
write!(out, "<pre>\n")?;
|
||||
write!(out, "METHOD {:?}<br>\n", req.method())?;
|
||||
write!(out, "URI {:?}<br>\n", req.uri())?;
|
||||
write!(out, "HOST {:?}<br>\n", req.uri().host())?;
|
||||
write!(out, "PORT {:?}<br>\n", req.uri().port())?;
|
||||
write!(out, "PATH {:?}<br>\n", req.uri().path())?;
|
||||
write!(out, "QUERY {:?}<br>\n", req.uri().query())?;
|
||||
for (hn, hv) in req.headers() {
|
||||
write!(out, "HEADER {hn:?}: {hv:?}<br>\n")?;
|
||||
}
|
||||
write!(out, "</pre>\n")?;
|
||||
Ok(response(StatusCode::NOT_FOUND).body(Body::from(body))?)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn proxy_distribute_v2(req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let path = req.uri().path();
|
||||
if path
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || ['/', '.', '-', '_'].contains(&c))
|
||||
&& !path.contains("..")
|
||||
{}
|
||||
if req.method() == Method::GET {
|
||||
let s = FileStream {
|
||||
file: File::open(format!("/opt/distri/{}", &path[DISTRI_PRE.len()..])).await?,
|
||||
};
|
||||
Ok(response(StatusCode::OK).body(Body::wrap_stream(s))?)
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FileStream {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl Stream for FileStream {
|
||||
type Item = Result<Vec<u8>, Error>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
use Poll::*;
|
||||
let mut buf = vec![0; 1024 * 8];
|
||||
let mut rb = ReadBuf::new(&mut buf);
|
||||
let f = &mut self.file;
|
||||
pin_mut!(f);
|
||||
match f.poll_read(cx, &mut rb) {
|
||||
Ready(k) => match k {
|
||||
Ok(_) => {
|
||||
let n = rb.filled().len();
|
||||
if n == 0 {
|
||||
Ready(None)
|
||||
} else {
|
||||
buf.truncate(n);
|
||||
Ready(Some(Ok(buf)))
|
||||
}
|
||||
}
|
||||
Err(e) => Ready(Some(Err(e.into()))),
|
||||
},
|
||||
Pending => Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct BackendsResponse {
|
||||
backends: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn backends(_req: Request<Body>, proxy_config: &ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
let backends: Vec<_> = proxy_config.backends.iter().map(|k| k.name.to_string()).collect();
|
||||
let res = BackendsResponse { backends };
|
||||
let ret = response(StatusCode::OK).body(Body::from(serde_json::to_vec(&res)?))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
match head.headers.get(http::header::ACCEPT) {
|
||||
Some(v) => {
|
||||
if v == APP_JSON {
|
||||
let url = Url::parse(&format!("dummy:{}", head.uri))?;
|
||||
let query = ChannelSearchQuery::from_url(&url)?;
|
||||
let mut methods = vec![];
|
||||
let mut bodies = vec![];
|
||||
let mut urls = proxy_config
|
||||
.backends
|
||||
.iter()
|
||||
.map(|sh| match Url::parse(&format!("{}/api/4/search/channel", sh.url)) {
|
||||
Ok(mut url) => {
|
||||
query.append_to_url(&mut url);
|
||||
Ok(url)
|
||||
}
|
||||
Err(_e) => Err(Error::with_msg(format!("parse error for: {:?}", sh))),
|
||||
})
|
||||
.fold_ok(vec![], |mut a, x| {
|
||||
a.push(x);
|
||||
methods.push(http::Method::GET);
|
||||
bodies.push(None);
|
||||
a
|
||||
})?;
|
||||
// TODO probably no longer needed?
|
||||
if let (Some(hosts), Some(backends)) = (None::<&Vec<String>>, None::<&Vec<String>>) {
|
||||
#[derive(Serialize)]
|
||||
struct QueryApi0 {
|
||||
backends: Vec<String>,
|
||||
regex: String,
|
||||
#[serde(rename = "sourceRegex")]
|
||||
source_regex: String,
|
||||
ordering: String,
|
||||
reload: bool,
|
||||
}
|
||||
hosts.iter().zip(backends.iter()).for_each(|(sh, back)| {
|
||||
let url = Url::parse(&format!("{}/channels/config", sh)).unwrap();
|
||||
urls.push(url);
|
||||
let q = QueryApi0 {
|
||||
backends: vec![back.into()],
|
||||
ordering: "asc".into(),
|
||||
reload: false,
|
||||
regex: query.name_regex.clone(),
|
||||
source_regex: query.source_regex.clone(),
|
||||
};
|
||||
let qs = serde_json::to_string(&q).unwrap();
|
||||
methods.push(http::Method::POST);
|
||||
bodies.push(Some(Body::from(qs)));
|
||||
});
|
||||
}
|
||||
let tags = urls.iter().map(|k| k.to_string()).collect();
|
||||
let nt = |tag, res| {
|
||||
let fut = async {
|
||||
let body = hyper::body::to_bytes(res).await?;
|
||||
//info!("got a result {:?}", body);
|
||||
let res: SubRes<ChannelSearchResult> =
|
||||
match serde_json::from_slice::<ChannelSearchResult>(&body) {
|
||||
Ok(val) => {
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val,
|
||||
};
|
||||
ret
|
||||
}
|
||||
Err(_) => {
|
||||
#[derive(Deserialize)]
|
||||
struct ResItemApi0 {
|
||||
name: String,
|
||||
source: String,
|
||||
backend: String,
|
||||
#[serde(rename = "type")]
|
||||
ty: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ResContApi0 {
|
||||
#[allow(dead_code)]
|
||||
backend: String,
|
||||
channels: Vec<ResItemApi0>,
|
||||
}
|
||||
match serde_json::from_slice::<Vec<ResContApi0>>(&body) {
|
||||
Ok(k) => {
|
||||
let mut a = vec![];
|
||||
if let Some(g) = k.first() {
|
||||
for c in &g.channels {
|
||||
let z = ChannelSearchSingleResult {
|
||||
backend: c.backend.clone(),
|
||||
description: String::new(),
|
||||
name: c.name.clone(),
|
||||
// TODO api 0 does not provide a series id
|
||||
series: 0,
|
||||
shape: vec![],
|
||||
source: c.source.clone(),
|
||||
ty: c.ty.clone(),
|
||||
unit: String::new(),
|
||||
is_api_0: Some(true),
|
||||
};
|
||||
a.push(z);
|
||||
}
|
||||
}
|
||||
let ret = ChannelSearchResult { channels: a };
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val: ret,
|
||||
};
|
||||
ret
|
||||
}
|
||||
Err(_) => {
|
||||
error!("Channel search response parse failed");
|
||||
let ret = ChannelSearchResult { channels: vec![] };
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val: ret,
|
||||
};
|
||||
ret
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
|
||||
};
|
||||
let ft = |all: Vec<(crate::gather::Tag, Result<SubRes<ChannelSearchResult>, Error>)>| {
|
||||
let mut res = Vec::new();
|
||||
for (_tag, j) in all {
|
||||
match j {
|
||||
Ok(j) => {
|
||||
for k in j.val.channels {
|
||||
res.push(k);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("{e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
let res = ChannelSearchResult { channels: res };
|
||||
let res = response(StatusCode::OK)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&res)?))?;
|
||||
Ok(res)
|
||||
};
|
||||
// TODO gather_get_json_generic must for this case accept a Method for each Request.
|
||||
// Currently it is inferred via presence of the body.
|
||||
// On the other hand, I want to gather over rather homogeneous requests.
|
||||
// So: better enforce same method.
|
||||
let ret = gather_get_json_generic(
|
||||
http::Method::GET,
|
||||
urls,
|
||||
bodies,
|
||||
tags,
|
||||
nt,
|
||||
ft,
|
||||
Duration::from_millis(3000),
|
||||
)
|
||||
.await?;
|
||||
Ok(ret)
|
||||
} else {
|
||||
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
None => Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn proxy_single_backend_query<QT>(
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
proxy_config: &ProxyConfig,
|
||||
) -> Result<Response<Body>, Error>
|
||||
where
|
||||
QT: FromUrl + AppendToUrl + HasBackend + HasTimeout,
|
||||
{
|
||||
let (head, _body) = req.into_parts();
|
||||
info!("proxy_single_backend_query {}", head.uri);
|
||||
match head.headers.get(http::header::ACCEPT) {
|
||||
Some(v) => {
|
||||
if v == APP_JSON || v == ACCEPT_ALL {
|
||||
let url = Url::parse(&format!("dummy:{}", head.uri))?;
|
||||
let query = match QT::from_url(&url) {
|
||||
Ok(k) => k,
|
||||
Err(_) => {
|
||||
let msg = format!("Malformed request or missing parameters");
|
||||
return Ok(response_err(StatusCode::BAD_REQUEST, msg)?);
|
||||
}
|
||||
};
|
||||
let sh = get_query_host_for_backend(&query.backend(), proxy_config)?;
|
||||
// TODO remove this special case
|
||||
// SPECIAL CASE:
|
||||
// Since the inner proxy is not yet handling map-pulse requests without backend,
|
||||
// we can not simply copy the original url here.
|
||||
// Instead, url needs to get parsed and formatted.
|
||||
// In general, the caller of this function should be able to provide a url, or maybe
|
||||
// better a closure so that the url can even depend on backend.
|
||||
let uri_path: String = if url.as_str().contains("/map/pulse/") {
|
||||
match MapPulseQuery::from_url(&url) {
|
||||
Ok(qu) => {
|
||||
info!("qu {qu:?}");
|
||||
format!("/api/4/map/pulse/{}/{}", qu.backend, qu.pulse)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("{e:?}");
|
||||
String::from("/BAD")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
head.uri.path().into()
|
||||
};
|
||||
info!("uri_path {uri_path}");
|
||||
let urls = [sh]
|
||||
.iter()
|
||||
.map(|sh| match Url::parse(&format!("{}{}", sh, uri_path)) {
|
||||
Ok(mut url) => {
|
||||
query.append_to_url(&mut url);
|
||||
Ok(url)
|
||||
}
|
||||
Err(e) => Err(Error::with_msg(format!("parse error for: {:?} {:?}", sh, e))),
|
||||
})
|
||||
.fold_ok(vec![], |mut a, x| {
|
||||
a.push(x);
|
||||
a
|
||||
})?;
|
||||
let tags: Vec<_> = urls.iter().map(|k| k.to_string()).collect();
|
||||
let nt = |tag: String, res: Response<Body>| {
|
||||
let fut = async {
|
||||
let (head, body) = res.into_parts();
|
||||
if head.status == StatusCode::OK {
|
||||
let body = hyper::body::to_bytes(body).await?;
|
||||
match serde_json::from_slice::<JsonValue>(&body) {
|
||||
Ok(val) => {
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: head.status,
|
||||
val,
|
||||
};
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("can not parse response: {e:?}");
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let body = hyper::body::to_bytes(body).await?;
|
||||
let b = String::from_utf8_lossy(&body);
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: head.status,
|
||||
// TODO would like to pass arbitrary type of body in these cases:
|
||||
val: serde_json::Value::String(format!("{}", b)),
|
||||
};
|
||||
Ok(ret)
|
||||
}
|
||||
};
|
||||
Box::pin(fut) as Pin<Box<dyn Future<Output = Result<SubRes<serde_json::Value>, Error>> + Send>>
|
||||
};
|
||||
let ft = |mut all: Vec<(crate::gather::Tag, Result<SubRes<JsonValue>, Error>)>| {
|
||||
if all.len() > 0 {
|
||||
all.truncate(1);
|
||||
let (_tag, z) = all.pop().unwrap();
|
||||
match z {
|
||||
Ok(z) => {
|
||||
let res = z.val;
|
||||
// TODO want to pass arbitrary body type:
|
||||
let res = response(z.status)
|
||||
.header(http::header::CONTENT_TYPE, APP_JSON)
|
||||
.body(Body::from(serde_json::to_string(&res)?))?;
|
||||
return Ok(res);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("FT sees: {e}");
|
||||
let res = crate::bodystream::ToPublicResponse::to_public_response(&e);
|
||||
return Ok(res);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err(Error::with_msg("no response from upstream"));
|
||||
}
|
||||
};
|
||||
let bodies = (0..urls.len()).into_iter().map(|_| None).collect();
|
||||
let ret =
|
||||
gather_get_json_generic(http::Method::GET, urls, bodies, tags, nt, ft, query.timeout()).await?;
|
||||
Ok(ret)
|
||||
} else {
|
||||
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
None => Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_query_host_for_backend(backend: &str, proxy_config: &ProxyConfig) -> Result<String, Error> {
|
||||
for back in &proxy_config.backends {
|
||||
if back.name == backend {
|
||||
return Ok(back.url.clone());
|
||||
}
|
||||
}
|
||||
return Err(Error::with_msg(format!("host not found for backend {:?}", backend)));
|
||||
}
|
||||
97
crates/httpret/src/proxy/api1.rs
Normal file
97
crates/httpret/src/proxy/api1.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
pub mod reqstatus;
|
||||
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::ReqCtx;
|
||||
use http::HeaderValue;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use hyper::Client;
|
||||
use netpod::log::*;
|
||||
use netpod::query::api1::Api1Query;
|
||||
use netpod::ProxyConfig;
|
||||
use netpod::ACCEPT_ALL;
|
||||
|
||||
pub struct PythonDataApi1Query {}
|
||||
|
||||
impl PythonDataApi1Query {
|
||||
pub fn path() -> &'static str {
|
||||
"/api/1/query"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == Self::path() {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
proxy_config: &ProxyConfig,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
if req.method() != Method::POST {
|
||||
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?);
|
||||
}
|
||||
let (head, body) = req.into_parts();
|
||||
let _accept = head
|
||||
.headers
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
|
||||
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
|
||||
.to_owned();
|
||||
let body_data = hyper::body::to_bytes(body).await?;
|
||||
if body_data.len() < 512 && body_data.first() == Some(&"{".as_bytes()[0]) {
|
||||
info!("request body_data string: {}", String::from_utf8_lossy(&body_data));
|
||||
}
|
||||
let qu = match serde_json::from_slice::<Api1Query>(&body_data) {
|
||||
Ok(qu) => qu,
|
||||
Err(e) => {
|
||||
error!("got body_data: {:?}", String::from_utf8_lossy(&body_data[..]));
|
||||
error!("can not parse: {e}");
|
||||
return Err(Error::with_msg_no_trace("can not parse query"));
|
||||
}
|
||||
};
|
||||
info!("Proxy sees request: {qu:?}");
|
||||
let back = {
|
||||
let mut ret = None;
|
||||
for b in &proxy_config.backends {
|
||||
if b.name == "sf-databuffer" {
|
||||
ret = Some(b);
|
||||
break;
|
||||
}
|
||||
}
|
||||
ret
|
||||
};
|
||||
if let Some(back) = back {
|
||||
let url_str = format!("{}/api/1/query", back.url);
|
||||
info!("try to ask {url_str}");
|
||||
let req = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(url_str)
|
||||
.body(Body::from(body_data))?;
|
||||
let client = Client::new();
|
||||
let res = client.request(req).await?;
|
||||
let (head, body) = res.into_parts();
|
||||
if head.status != StatusCode::OK {
|
||||
error!("backend returned error: {head:?}");
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?)
|
||||
} else {
|
||||
info!("backend returned OK");
|
||||
let riq_def = HeaderValue::from_static("(none)");
|
||||
let riq = head.headers.get("x-daqbuffer-request-id").unwrap_or(&riq_def);
|
||||
Ok(response(StatusCode::OK)
|
||||
.header("x-daqbuffer-request-id", riq)
|
||||
.body(body)?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
81
crates/httpret/src/proxy/api1/reqstatus.rs
Normal file
81
crates/httpret/src/proxy/api1/reqstatus.rs
Normal file
@@ -0,0 +1,81 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use hyper::Client;
|
||||
use netpod::log::*;
|
||||
use netpod::ProxyConfig;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
|
||||
pub struct RequestStatusHandler {}
|
||||
|
||||
impl RequestStatusHandler {
|
||||
pub fn path_prefix() -> &'static str {
|
||||
"/api/1/requestStatus/"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path().starts_with(Self::path_prefix()) {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, proxy_config: &ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
let (head, body) = req.into_parts();
|
||||
if head.method != Method::GET {
|
||||
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?);
|
||||
}
|
||||
let accept = head
|
||||
.headers
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
|
||||
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
|
||||
.to_owned();
|
||||
if accept != APP_JSON && accept != ACCEPT_ALL {
|
||||
// TODO set the public error code and message and return Err(e).
|
||||
let e = Error::with_public_msg(format!("Unsupported Accept: {:?}", accept));
|
||||
error!("{e}");
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
let _body_data = hyper::body::to_bytes(body).await?;
|
||||
let status_id = &head.uri.path()[Self::path_prefix().len()..];
|
||||
info!("RequestStatusHandler status_id {:?}", status_id);
|
||||
|
||||
let back = {
|
||||
let mut ret = None;
|
||||
for b in &proxy_config.backends {
|
||||
if b.name == "sf-databuffer" {
|
||||
ret = Some(b);
|
||||
break;
|
||||
}
|
||||
}
|
||||
ret
|
||||
};
|
||||
if let Some(back) = back {
|
||||
let url_str = format!("{}{}{}", back.url, Self::path_prefix(), status_id);
|
||||
info!("try to ask {url_str}");
|
||||
let req = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(url_str)
|
||||
.body(Body::empty())?;
|
||||
let client = Client::new();
|
||||
let res = client.request(req).await?;
|
||||
let (head, body) = res.into_parts();
|
||||
if head.status != StatusCode::OK {
|
||||
error!("backend returned error: {head:?}");
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?)
|
||||
} else {
|
||||
info!("backend returned OK");
|
||||
Ok(response(StatusCode::OK).body(body)?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
268
crates/httpret/src/proxy/api4.rs
Normal file
268
crates/httpret/src/proxy/api4.rs
Normal file
@@ -0,0 +1,268 @@
|
||||
pub mod caioclookup;
|
||||
|
||||
use crate::bodystream::ToPublicResponse;
|
||||
use crate::err::Error;
|
||||
use crate::gather::gather_get_json_generic;
|
||||
use crate::gather::SubRes;
|
||||
use crate::gather::Tag;
|
||||
use crate::response;
|
||||
use crate::ReqCtx;
|
||||
use futures_util::Future;
|
||||
use http::Method;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::ChannelSearchQuery;
|
||||
use netpod::ChannelSearchResult;
|
||||
use netpod::NodeStatus;
|
||||
use netpod::NodeStatusSub;
|
||||
use netpod::ProxyConfig;
|
||||
use netpod::ACCEPT_ALL;
|
||||
use netpod::APP_JSON;
|
||||
use serde_json::Value as JsVal;
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
// TODO model channel search according to StatusNodesRecursive.
|
||||
// Make sure that backend handling is correct:
|
||||
// The aggregator asks all backends, except if the user specifies some backend
|
||||
// in which case it should only go to the matching backends.
|
||||
// The aggregators and leaf nodes behind should as well not depend on backend,
|
||||
// but simply answer all matching.
|
||||
|
||||
pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> Result<ChannelSearchResult, Error> {
|
||||
let (head, _body) = req.into_parts();
|
||||
let inpurl = Url::parse(&format!("dummy:{}", head.uri))?;
|
||||
let query = ChannelSearchQuery::from_url(&inpurl)?;
|
||||
let mut urls = Vec::new();
|
||||
let mut tags = Vec::new();
|
||||
let mut bodies = Vec::new();
|
||||
for pb in &proxy_config.backends {
|
||||
if if let Some(b) = &query.backend {
|
||||
pb.name.contains(b)
|
||||
} else {
|
||||
true
|
||||
} {
|
||||
match Url::parse(&format!("{}/api/4/search/channel", pb.url)) {
|
||||
Ok(mut url) => {
|
||||
query.append_to_url(&mut url);
|
||||
tags.push(url.to_string());
|
||||
bodies.push(None);
|
||||
urls.push(url);
|
||||
}
|
||||
Err(_) => return Err(Error::with_msg(format!("parse error for: {:?}", pb))),
|
||||
}
|
||||
}
|
||||
}
|
||||
let nt = |tag, res| {
|
||||
let fut = async {
|
||||
let body = hyper::body::to_bytes(res).await?;
|
||||
//info!("got a result {:?}", body);
|
||||
let res: ChannelSearchResult = match serde_json::from_slice(&body) {
|
||||
Ok(k) => k,
|
||||
Err(_) => {
|
||||
let msg = format!("can not parse result: {}", String::from_utf8_lossy(&body));
|
||||
error!("{}", msg);
|
||||
return Err(Error::with_msg_no_trace(msg));
|
||||
}
|
||||
};
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val: res,
|
||||
};
|
||||
Ok(ret)
|
||||
};
|
||||
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
|
||||
};
|
||||
let ft = |all: Vec<(Tag, Result<SubRes<ChannelSearchResult>, Error>)>| {
|
||||
let mut res = Vec::new();
|
||||
for (_tag, j) in all {
|
||||
match j {
|
||||
Ok(j) => {
|
||||
for k in j.val.channels {
|
||||
res.push(k);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("{e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
let res = ChannelSearchResult { channels: res };
|
||||
Ok(res)
|
||||
};
|
||||
let ret = gather_get_json_generic(
|
||||
http::Method::GET,
|
||||
urls,
|
||||
bodies,
|
||||
tags,
|
||||
nt,
|
||||
ft,
|
||||
Duration::from_millis(3000),
|
||||
)
|
||||
.await?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub struct ChannelSearchAggHandler {}
|
||||
|
||||
impl ChannelSearchAggHandler {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == "/api/4/search/channel" {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &ProxyConfig) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
let accept_def = APP_JSON;
|
||||
let accept = req
|
||||
.headers()
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
|
||||
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
|
||||
match channel_search(req, node_config).await {
|
||||
Ok(item) => {
|
||||
let buf = serde_json::to_vec(&item)?;
|
||||
Ok(response(StatusCode::OK).body(Body::from(buf))?)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("ChannelConfigHandler::handle: got error from channel_config: {e:?}");
|
||||
Ok(e.to_public_response())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
|
||||
}
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StatusNodesRecursive {}
|
||||
|
||||
impl StatusNodesRecursive {
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == crate::api4::status::StatusNodesRecursive::path() {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
node_config: &ProxyConfig,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match self.status(req, ctx, node_config).await {
|
||||
Ok(status) => {
|
||||
let body = serde_json::to_vec(&status)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(body))?;
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("StatusNodesRecursive sees: {e}");
|
||||
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn status(
|
||||
&self,
|
||||
_req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
proxy_config: &ProxyConfig,
|
||||
) -> Result<NodeStatus, Error> {
|
||||
let path = crate::api4::status::StatusNodesRecursive::path();
|
||||
let mut bodies = Vec::new();
|
||||
let mut urls = Vec::new();
|
||||
let mut tags = Vec::new();
|
||||
for sub in &proxy_config.status_subs {
|
||||
match Url::parse(&format!("{}{}", sub.url, path)) {
|
||||
Ok(url) => {
|
||||
bodies.push(None);
|
||||
tags.push(sub.url.to_string());
|
||||
urls.push(url);
|
||||
}
|
||||
Err(e) => return Err(Error::with_msg_no_trace(format!("parse error for: {sub:?} {e:?}"))),
|
||||
}
|
||||
}
|
||||
let nt = |tag, res| {
|
||||
let fut = async {
|
||||
let body = hyper::body::to_bytes(res).await?;
|
||||
let res: JsVal = match serde_json::from_slice(&body) {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
error!("{e}");
|
||||
let msg = format!(
|
||||
"gather sub responses, can not parse result: {} {}",
|
||||
String::from_utf8_lossy(&body),
|
||||
e,
|
||||
);
|
||||
error!("{}", msg);
|
||||
return Err(Error::with_msg_no_trace(msg));
|
||||
}
|
||||
};
|
||||
let ret = SubRes {
|
||||
tag,
|
||||
status: StatusCode::OK,
|
||||
val: res,
|
||||
};
|
||||
Ok(ret)
|
||||
};
|
||||
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
|
||||
};
|
||||
let ft = |all: Vec<(Tag, Result<SubRes<JsVal>, Error>)>| {
|
||||
let mut subs = VecDeque::new();
|
||||
for (tag, sr) in all {
|
||||
match sr {
|
||||
Ok(sr) => {
|
||||
let s: Result<NodeStatus, _> = serde_json::from_value(sr.val).map_err(err::Error::from);
|
||||
let sub = NodeStatusSub { url: tag.0, status: s };
|
||||
subs.push_back(sub);
|
||||
}
|
||||
Err(e) => {
|
||||
let sub = NodeStatusSub {
|
||||
url: tag.0,
|
||||
status: Err(err::Error::from(e)),
|
||||
};
|
||||
subs.push_back(sub);
|
||||
}
|
||||
}
|
||||
}
|
||||
let ret = NodeStatus {
|
||||
name: format!("{}:{}", proxy_config.name, proxy_config.port),
|
||||
version: core::env!("CARGO_PKG_VERSION").into(),
|
||||
is_sf_databuffer: false,
|
||||
is_archiver_engine: false,
|
||||
is_archiver_appliance: false,
|
||||
database_size: None,
|
||||
archiver_appliance_status: None,
|
||||
subs,
|
||||
};
|
||||
Ok(ret)
|
||||
};
|
||||
let ret = gather_get_json_generic(
|
||||
http::Method::GET,
|
||||
urls,
|
||||
bodies,
|
||||
tags,
|
||||
nt,
|
||||
ft,
|
||||
Duration::from_millis(1200),
|
||||
)
|
||||
.await?;
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
54
crates/httpret/src/proxy/api4/caioclookup.rs
Normal file
54
crates/httpret/src/proxy/api4/caioclookup.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
use crate::bodystream::response;
|
||||
use crate::err::Error;
|
||||
use crate::ReqCtx;
|
||||
use http::Request;
|
||||
use http::Response;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::ProxyConfig;
|
||||
|
||||
pub struct CaIocLookup {}
|
||||
|
||||
impl CaIocLookup {
|
||||
fn path() -> &'static str {
|
||||
"/api/4/channel-access/search/addr"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path() == Self::path() {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
ctx: &ReqCtx,
|
||||
node_config: &ProxyConfig,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match self.search(req, ctx, node_config).await {
|
||||
Ok(status) => {
|
||||
let body = serde_json::to_vec(&status)?;
|
||||
let ret = response(StatusCode::OK).body(Body::from(body))?;
|
||||
Ok(ret)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("sees: {e}");
|
||||
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn search(
|
||||
&self,
|
||||
_req: Request<Body>,
|
||||
_ctx: &ReqCtx,
|
||||
_proxy_config: &ProxyConfig,
|
||||
) -> Result<Option<String>, Error> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
1507
crates/httpret/src/pulsemap.rs
Normal file
1507
crates/httpret/src/pulsemap.rs
Normal file
File diff suppressed because it is too large
Load Diff
62
crates/httpret/src/settings.rs
Normal file
62
crates/httpret/src/settings.rs
Normal file
@@ -0,0 +1,62 @@
|
||||
use crate::err::Error;
|
||||
use crate::response;
|
||||
use http::{Method, StatusCode};
|
||||
use hyper::{Body, Request, Response};
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::{ACCEPT_ALL, APP_JSON};
|
||||
|
||||
pub struct SettingsThreadsMaxHandler {}
|
||||
|
||||
impl SettingsThreadsMaxHandler {
|
||||
pub fn path_prefix() -> &'static str {
|
||||
"/api/4/settings/read3/threads_max"
|
||||
}
|
||||
|
||||
pub fn handler(req: &Request<Body>) -> Option<Self> {
|
||||
if req.uri().path().starts_with(Self::path_prefix()) {
|
||||
Some(Self {})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn put(&self, req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let (head, body) = req.into_parts();
|
||||
let accept = head
|
||||
.headers
|
||||
.get(http::header::ACCEPT)
|
||||
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
|
||||
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
|
||||
.to_owned();
|
||||
if accept != APP_JSON && accept != ACCEPT_ALL {
|
||||
// TODO set the public error code and message and return Err(e).
|
||||
let e = Error::with_public_msg(format!("Unsupported Accept: {:?}", accept));
|
||||
error!("{e}");
|
||||
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
|
||||
}
|
||||
let body = hyper::body::to_bytes(body).await?;
|
||||
//let threads_max: usize = head.uri.path()[Self::path_prefix().len()..].parse()?;
|
||||
let threads_max: usize = String::from_utf8_lossy(&body).parse()?;
|
||||
info!("threads_max {threads_max}");
|
||||
disk::read3::Read3::get().set_threads_max(threads_max);
|
||||
let ret = response(StatusCode::NO_CONTENT).body(Body::empty())?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub async fn get(&self, _req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
let threads_max = disk::read3::Read3::get().threads_max();
|
||||
let ret = response(StatusCode::OK).body(Body::from(format!("{threads_max}")))?;
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
|
||||
if req.method() == Method::GET {
|
||||
self.get(req, node_config).await
|
||||
} else if req.method() == Method::PUT {
|
||||
self.put(req, node_config).await
|
||||
} else {
|
||||
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user