daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Start improving client facing errors

Browse Source
This commit is contained in:
Dominik Werder
2024-08-20 16:16:43 +02:00
parent cb92317bf6
commit 7a8d071c7a
23 changed files with 984 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
crates/httpret/src/api4/binned.rs
View File

@@ -1,15 +1,16 @@
use crate::bodystream::response;
use crate::bodystream::response_err_msg;
use crate::bodystream::ToPublicResponse;
use crate::channelconfig::ch_conf_from_binned;
use crate::err::Error;
use crate::requests::accepts_json_or_all;
use crate::requests::accepts_octets;
use crate::ServiceSharedResources;
use dbconn::worker::PgQueue;
use err::thiserror;
use err::ThisError;
use http::Method;
use http::StatusCode;
use httpclient::body_empty;
use httpclient::error_response;
use httpclient::not_found_response;
use httpclient::IntoBody;
use httpclient::Requ;
use httpclient::StreamResponse;
@@ -25,69 +26,27 @@ use query::api4::binned::BinnedQuery;
use tracing::Instrument;
use url::Url;
async fn binned_json(
url: Url,
req: Requ,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
debug!("{:?}", req);
let reqid = crate::status_board()
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?
.new_status_id();
let (_head, _body) = req.into_parts();
let query = BinnedQuery::from_url(&url).map_err(|e| {
error!("binned_json: {e:?}");
let msg = format!("can not parse query: {}", e.msg());
e.add_public_msg(msg)
})?;
// TODO handle None case better and return 404
let ch_conf = ch_conf_from_binned(&query, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
let span1 = span!(
Level::INFO,
"httpret::binned",
reqid,
beg = query.range().beg_u64() / SEC,
end = query.range().end_u64() / SEC,
ch = query.channel().name(),
);
span1.in_scope(|| {
debug!("begin");
});
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let open_bytes = Box::pin(open_bytes);
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, ctx, open_bytes)
.instrument(span1)
.await?;
let ret = response(StatusCode::OK).body(ToJsonBody::from(&item).into_body())?;
Ok(ret)
#[derive(Debug, ThisError)]
#[cstm(name = "Api4Binned")]
pub enum Error {
ChannelNotFound,
BadQuery(String),
HttpLib(#[from] http::Error),
ChannelConfig(crate::channelconfig::Error),
Retrieval(#[from] crate::RetrievalError),
EventsCbor(#[from] streams::plaineventscbor::Error),
EventsJson(#[from] streams::plaineventsjson::Error),
ServerError,
BinnedStream(::err::Error),
}
async fn binned(req: Requ, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri())?;
if req
.uri()
.path_and_query()
.map_or(false, |x| x.as_str().contains("DOERR"))
{
Err(Error::with_msg_no_trace("hidden message").add_public_msg("PublicMessage"))?;
}
if accepts_json_or_all(&req.headers()) {
Ok(binned_json(url, req, ctx, pgqueue, ncc).await?)
} else if accepts_octets(&req.headers()) {
Ok(response_err_msg(
StatusCode::NOT_ACCEPTABLE,
format!("binary binned data not yet available"),
)?)
} else {
let ret = response_err_msg(
StatusCode::NOT_ACCEPTABLE,
format!("Unsupported Accept: {:?}", req.headers()),
)?;
Ok(ret)
impl From<crate::channelconfig::Error> for Error {
fn from(value: crate::channelconfig::Error) -> Self {
use crate::channelconfig::Error::*;
match value {
NotFound(_) => Self::ChannelNotFound,
_ => Self::ChannelConfig(value),
}
}
}
@@ -110,14 +69,86 @@ impl BinnedHandler {
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(body_empty())?);
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?);
}
match binned(req, ctx, &shared_res.pgqueue, ncc).await {
Ok(ret) => Ok(ret),
Err(e) => {
warn!("BinnedHandler handle sees: {e}");
Ok(e.to_public_response())
}
Err(e) => match e {
Error::ChannelNotFound => {
let res = not_found_response("channel not found".into(), ctx.reqid());
Ok(res)
}
Error::BadQuery(msg) => {
let res = error_response(format!("bad query: {msg}"), ctx.reqid());
Ok(res)
}
_ => {
error!("EventsHandler sees: {e}");
Ok(error_response(e.public_message(), ctx.reqid()))
}
},
}
}
}
async fn binned(req: Requ, ctx: &ReqCtx, pgqueue: &PgQueue, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri()).map_err(|e| Error::BadQuery(e.to_string()))?;
if req
.uri()
.path_and_query()
.map_or(false, |x| x.as_str().contains("DOERR"))
{
Err(Error::ServerError)?;
}
if accepts_json_or_all(&req.headers()) {
Ok(binned_json(url, req, ctx, pgqueue, ncc).await?)
} else if accepts_octets(&req.headers()) {
Ok(error_response(
format!("binary binned data not yet available"),
ctx.reqid(),
))
} else {
let ret = error_response(format!("Unsupported Accept: {:?}", req.headers()), ctx.reqid());
Ok(ret)
}
}
async fn binned_json(
url: Url,
req: Requ,
ctx: &ReqCtx,
pgqueue: &PgQueue,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
debug!("{:?}", req);
let reqid = crate::status_board().map_err(|_e| Error::ServerError)?.new_status_id();
let (_head, _body) = req.into_parts();
let query = BinnedQuery::from_url(&url).map_err(|e| {
error!("binned_json: {e:?}");
Error::BadQuery(e.to_string())
})?;
// TODO handle None case better and return 404
let ch_conf = ch_conf_from_binned(&query, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::ChannelNotFound)?;
let span1 = span!(
Level::INFO,
"httpret::binned",
reqid,
beg = query.range().beg_u64() / SEC,
end = query.range().end_u64() / SEC,
ch = query.channel().name(),
);
span1.in_scope(|| {
debug!("begin");
});
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let open_bytes = Box::pin(open_bytes);
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, ctx, open_bytes)
.instrument(span1)
.await
.map_err(|e| Error::BinnedStream(e))?;
let ret = response(StatusCode::OK).body(ToJsonBody::from(&item).into_body())?;
// let ret = error_response(e.public_message(), ctx.reqid());
Ok(ret)
}

5
crates/httpret/src/api4/databuffer_tools.rs
View File

@@ -1,5 +1,4 @@
use crate::bodystream::response;
use crate::bodystream::response_err_msg;
use async_channel::Receiver;
use async_channel::Sender;
use bytes::Bytes;
@@ -14,6 +13,7 @@ use http::Response;
use http::StatusCode;
use httpclient::body_empty;
use httpclient::body_stream;
use httpclient::error_response;
use httpclient::Requ;
use httpclient::StreamResponse;
use netpod::log::*;
@@ -76,8 +76,7 @@ impl FindActiveHandler {
Ok(ret) => Ok(ret),
Err(e) => {
error!("{e}");
let res = response_err_msg(StatusCode::NOT_ACCEPTABLE, e.to_public_error())
.map_err(|_| FindActiveError::InternalError)?;
let res = error_response(e.to_public_error().to_string(), "missing-req");
Ok(res)
}
}

7
crates/httpret/src/api4/eventdata.rs
View File

@@ -1,4 +1,3 @@
use crate::bodystream::response_err_msg;
use crate::response;
use crate::ReqCtx;
use crate::ServiceSharedResources;
@@ -10,6 +9,7 @@ use http::Method;
use http::StatusCode;
use httpclient::body_empty;
use httpclient::body_stream;
use httpclient::error_response;
use httpclient::read_body_bytes;
use httpclient::Requ;
use httpclient::StreamResponse;
@@ -50,7 +50,7 @@ impl EventDataHandler {
pub async fn handle(
&self,
req: Requ,
_ctx: &ReqCtx,
ctx: &ReqCtx,
ncc: &NodeConfigCached,
shared_res: Arc<ServiceSharedResources>,
) -> Result<StreamResponse, EventDataError> {
@@ -63,8 +63,7 @@ impl EventDataHandler {
Ok(ret) => Ok(ret),
Err(e) => {
error!("{e}");
let res = response_err_msg(StatusCode::NOT_ACCEPTABLE, e.to_public_error())
.map_err(|_| EventDataError::InternalError)?;
let res = error_response(e.to_public_error().to_string(), ctx.reqid());
Ok(res)
}
}

84
crates/httpret/src/api4/events.rs
View File

@@ -1,23 +1,25 @@
use crate::bodystream::response_err_msg;
use crate::channelconfig::chconf_from_events_quorum;
use crate::err::Error;
use crate::requests::accepts_cbor_framed;
use crate::requests::accepts_json_framed;
use crate::requests::accepts_json_or_all;
use crate::response;
use crate::ServiceSharedResources;
use crate::ToPublicResponse;
use bytes::Bytes;
use bytes::BytesMut;
use dbconn::worker::PgQueue;
use err::thiserror;
use err::ThisError;
use futures_util::future;
use futures_util::stream;
use futures_util::Stream;
use futures_util::StreamExt;
use http::header::CONTENT_TYPE;
use http::Method;
use http::StatusCode;
use httpclient::body_empty;
use httpclient::body_stream;
use httpclient::error_response;
use httpclient::not_found_response;
use httpclient::IntoBody;
use httpclient::Requ;
use httpclient::StreamResponse;
@@ -28,11 +30,36 @@ use netpod::ChannelTypeConfigGen;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use netpod::ReqCtx;
use netpod::APP_CBOR_FRAMED;
use netpod::APP_JSON;
use netpod::APP_JSON_FRAMED;
use netpod::HEADER_NAME_REQUEST_ID;
use nodenet::client::OpenBoxedBytesViaHttp;
use query::api4::events::PlainEventsQuery;
use streams::instrument::InstrumentStream;
use tracing::Instrument;
#[derive(Debug, ThisError)]
#[cstm(name = "Api4Events")]
pub enum Error {
ChannelNotFound,
HttpLib(#[from] http::Error),
ChannelConfig(crate::channelconfig::Error),
Retrieval(#[from] crate::RetrievalError),
EventsCbor(#[from] streams::plaineventscbor::Error),
EventsJson(#[from] streams::plaineventsjson::Error),
}
impl From<crate::channelconfig::Error> for Error {
fn from(value: crate::channelconfig::Error) -> Self {
use crate::channelconfig::Error::*;
match value {
NotFound(_) => Self::ChannelNotFound,
_ => Self::ChannelConfig(value),
}
}
}
pub struct EventsHandler {}
impl EventsHandler {
@@ -50,9 +77,9 @@ impl EventsHandler {
ctx: &ReqCtx,
shared_res: &ServiceSharedResources,
ncc: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
) -> Result<StreamResponse, crate::err::Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(body_empty())?);
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?);
}
let self_name = "handle";
let url = req_uri_to_url(req.uri())?;
@@ -73,10 +100,16 @@ impl EventsHandler {
.await
{
Ok(ret) => Ok(ret),
Err(e) => {
error!("EventsHandler sees: {e}");
Ok(e.to_public_response())
}
Err(e) => match e {
Error::ChannelNotFound => {
let res = not_found_response("channel not found".into(), ctx.reqid());
Ok(res)
}
_ => {
error!("EventsHandler sees: {e}");
Ok(error_response(e.public_message(), ctx.reqid()))
}
},
}
}
}
@@ -90,7 +123,7 @@ async fn plain_events(
) -> Result<StreamResponse, Error> {
let ch_conf = chconf_from_events_quorum(&evq, ctx, pgqueue, ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
.ok_or_else(|| Error::ChannelNotFound)?;
if accepts_cbor_framed(req.headers()) {
Ok(plain_events_cbor_framed(req, evq, ch_conf, ctx, ncc).await?)
} else if accepts_json_framed(req.headers()) {
@@ -98,7 +131,7 @@ async fn plain_events(
} else if accepts_json_or_all(req.headers()) {
Ok(plain_events_json(req, evq, ch_conf, ctx, ncc).await?)
} else {
let ret = response_err_msg(StatusCode::NOT_ACCEPTABLE, format!("unsupported accept {:?}", req))?;
let ret = error_response(format!("unsupported accept"), ctx.reqid());
Ok(ret)
}
}
@@ -124,7 +157,10 @@ async fn plain_events_cbor_framed(
tracing::Span::none()
};
let stream = InstrumentStream::new(stream, logspan);
let ret = response(StatusCode::OK).body(body_stream(stream))?;
let ret = response(StatusCode::OK)
.header(CONTENT_TYPE, APP_CBOR_FRAMED)
.header(HEADER_NAME_REQUEST_ID, ctx.reqid())
.body(body_stream(stream))?;
Ok(ret)
}
@@ -139,7 +175,10 @@ async fn plain_events_json_framed(
let open_bytes = OpenBoxedBytesViaHttp::new(ncc.node_config.cluster.clone());
let stream = streams::plaineventsjson::plain_events_json_stream(&evq, ch_conf, ctx, Box::pin(open_bytes)).await?;
let stream = bytes_chunks_to_len_framed_str(stream);
let ret = response(StatusCode::OK).body(body_stream(stream))?;
let ret = response(StatusCode::OK)
.header(CONTENT_TYPE, APP_JSON_FRAMED)
.header(HEADER_NAME_REQUEST_ID, ctx.reqid())
.body(body_stream(stream))?;
Ok(ret)
}
@@ -167,12 +206,15 @@ async fn plain_events_json(
return Err(e.into());
}
};
let ret = response(StatusCode::OK).body(ToJsonBody::from(&item).into_body())?;
let ret = response(StatusCode::OK)
.header(CONTENT_TYPE, APP_JSON)
.header(HEADER_NAME_REQUEST_ID, ctx.reqid())
.body(ToJsonBody::from(&item).into_body())?;
debug!("{self_name} response created");
Ok(ret)
}
fn bytes_chunks_to_framed<S, T>(stream: S) -> impl Stream<Item = Result<Bytes, Error>>
fn bytes_chunks_to_framed<S, T>(stream: S) -> impl Stream<Item = Result<Bytes, crate::err::Error>>
where
S: Stream<Item = Result<T, err::Error>>,
T: Into<Bytes>,
@@ -191,19 +233,19 @@ where
b2.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
let mut b3 = BytesMut::with_capacity(16);
b3.put_slice(&[0, 0, 0, 0, 0, 0, 0, 0][..pad]);
stream::iter([Ok::<_, Error>(b2.freeze()), Ok(buf), Ok(b3.freeze())])
stream::iter([Ok::<_, crate::err::Error>(b2.freeze()), Ok(buf), Ok(b3.freeze())])
}
Err(e) => {
let e = Error::with_msg_no_trace(e.to_string());
let e = crate::err::Error::with_msg_no_trace(e.to_string());
stream::iter([Err(e), Ok(Bytes::new()), Ok(Bytes::new())])
}
})
.filter(|x| if let Ok(x) = x { ready(x.len() > 0) } else { ready(true) })
}
fn bytes_chunks_to_len_framed_str<S, T>(stream: S) -> impl Stream<Item = Result<String, Error>>
fn bytes_chunks_to_len_framed_str<S, T>(stream: S) -> impl Stream<Item = Result<String, crate::err::Error>>
where
S: Stream<Item = Result<T, err::Error>>,
S: Stream<Item = Result<T, ::err::Error>>,
T: Into<String>,
{
use future::ready;
@@ -214,10 +256,10 @@ where
let s = y.into();
let mut b2 = String::with_capacity(16);
write!(b2, "\n{}\n", s.len()).unwrap();
stream::iter([Ok::<_, Error>(b2), Ok(s)])
stream::iter([Ok::<_, crate::err::Error>(b2), Ok(s)])
}
Err(e) => {
let e = Error::with_msg_no_trace(e.to_string());
let e = crate::err::Error::with_msg_no_trace(e.to_string());
stream::iter([Err(e), Ok(String::new())])
}
})

8
crates/httpret/src/bodystream.rs
View File

@@ -17,14 +17,6 @@ where
Response::builder().status(status)
}
pub fn response_err_msg<T>(status: StatusCode, msg: T) -> Result<StreamResponse, RetrievalError>
where
T: ToString,
{
let ret = response(status).body(body_string(msg))?;
Ok(ret)
}
pub trait ToPublicResponse {
fn to_public_response(&self) -> StreamResponse;
}

285
crates/httpret/src/channelconfig.rs
View File

@@ -1,7 +1,6 @@
use crate::err::Error;
use crate::response;
use crate::ServiceSharedResources;
use crate::ToPublicResponse;
use core::fmt;
use dbconn::create_connection;
use dbconn::worker::PgQueue;
use futures_util::StreamExt;
@@ -36,12 +35,191 @@ use netpod::APP_JSON;
use nodenet::configquorum::find_config_basics_quorum;
use query::api4::binned::BinnedQuery;
use query::api4::events::PlainEventsQuery;
use scyllaconn::errconv::ErrConv;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
use url::Url;
#[derive(Debug)]
pub enum Error {
NotFound(SfDbChannel),
ConfigQuorum(nodenet::configquorum::Error),
ConfigNode(nodenet::channelconfig::Error),
Http(crate::Error),
HttpCrate(http::Error),
// TODO create dedicated error type for query parsing
BadQuery(err::Error),
MissingBackend,
MissingScalarType,
MissingShape,
MissingShapeKind,
MissingEdge,
Uri(netpod::UriError),
ChannelConfigQuery(err::Error),
ExpectScyllaBackend,
Pg(dbconn::pg::Error),
Scylla(String),
Join,
OtherErr(err::Error),
PgWorker(dbconn::worker::Error),
Async(netpod::AsyncChannelError),
ChannelConfig(dbconn::channelconfig::Error),
}
impl fmt::Display for Error {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
let name = "HttpChannelConfigError";
write!(fmt, "{name}(")?;
match self {
Error::NotFound(chn) => write!(fmt, "NotFound({chn}")?,
Error::ConfigQuorum(e) => write!(fmt, "ConfigQuorum({e})")?,
Error::ConfigNode(e) => write!(fmt, "ConfigNode({e})")?,
Error::Http(e) => write!(fmt, "Http({e})")?,
Error::HttpCrate(e) => write!(fmt, "HttpCrate({e})")?,
Error::BadQuery(e) => write!(fmt, "BadQuery({e})")?,
Error::MissingBackend => write!(fmt, "MissingBackend")?,
Error::MissingScalarType => write!(fmt, "MissingScalarType")?,
Error::MissingShape => write!(fmt, "MissingShape")?,
Error::MissingShapeKind => write!(fmt, "MissingShapeKind")?,
Error::MissingEdge => write!(fmt, "MissingEdge")?,
Error::Uri(x) => write!(fmt, "Uri({x})")?,
Error::ChannelConfigQuery(e) => write!(fmt, "ChannelConfigQuery({e})")?,
Error::ExpectScyllaBackend => write!(fmt, "ExpectScyllaBackend")?,
Error::Pg(e) => write!(fmt, "Pg({e})")?,
Error::Scylla(e) => write!(fmt, "Scylla({e})")?,
Error::Join => write!(fmt, "Join")?,
Error::OtherErr(e) => write!(fmt, "OtherErr({e})")?,
Error::PgWorker(e) => write!(fmt, "PgWorker({e})")?,
Error::Async(e) => write!(fmt, "Async({e})")?,
Error::ChannelConfig(e) => write!(fmt, "ChannelConfig({e})")?,
}
write!(fmt, ")")?;
Ok(())
}
}
fn other_err_error(e: err::Error) -> Error {
Error::OtherErr(e)
}
impl std::error::Error for Error {}
impl From<crate::Error> for Error {
fn from(e: crate::Error) -> Self {
Self::Http(e)
}
}
impl From<http::Error> for Error {
fn from(e: http::Error) -> Self {
Self::HttpCrate(e)
}
}
impl From<nodenet::configquorum::Error> for Error {
fn from(e: nodenet::configquorum::Error) -> Self {
use nodenet::configquorum::Error::*;
match e {
NotFound(a) => Self::NotFound(a),
_ => Self::ConfigQuorum(e),
}
}
}
impl From<nodenet::channelconfig::Error> for Error {
fn from(e: nodenet::channelconfig::Error) -> Self {
match e {
nodenet::channelconfig::Error::NotFoundChannel(a) => Self::NotFound(a),
_ => Self::ConfigNode(e),
}
}
}
impl From<netpod::UriError> for Error {
fn from(e: netpod::UriError) -> Self {
Self::Uri(e)
}
}
impl From<dbconn::pg::Error> for Error {
fn from(e: dbconn::pg::Error) -> Self {
Self::Pg(e)
}
}
impl From<dbconn::worker::Error> for Error {
fn from(e: dbconn::worker::Error) -> Self {
Self::PgWorker(e)
}
}
impl From<scyllaconn::scylla::cql_to_rust::FromRowError> for Error {
fn from(e: scyllaconn::scylla::cql_to_rust::FromRowError) -> Self {
Self::Scylla(e.to_string())
}
}
impl From<scyllaconn::scylla::transport::errors::QueryError> for Error {
fn from(e: scyllaconn::scylla::transport::errors::QueryError) -> Self {
Self::Scylla(e.to_string())
}
}
impl From<scyllaconn::scylla::transport::iterator::NextRowError> for Error {
fn from(e: scyllaconn::scylla::transport::iterator::NextRowError) -> Self {
Self::Scylla(e.to_string())
}
}
impl From<taskrun::tokio::task::JoinError> for Error {
fn from(_e: taskrun::tokio::task::JoinError) -> Self {
Self::Join
}
}
impl From<netpod::AsyncChannelError> for Error {
fn from(e: netpod::AsyncChannelError) -> Self {
Self::Async(e)
}
}
impl From<dbconn::channelconfig::Error> for Error {
fn from(e: dbconn::channelconfig::Error) -> Self {
Self::ChannelConfig(e)
}
}
impl From<Error> for crate::err::Error {
fn from(e: Error) -> Self {
Self::with_msg_no_trace(format!("{e} TODO add public message"))
}
}
impl Error {
fn to_public_response(self) -> http::Response<httpclient::StreamBody> {
use httpclient::internal_error;
let status = StatusCode::INTERNAL_SERVER_ERROR;
let js = serde_json::json!({
"message": self.to_string(),
});
if let Ok(body) = serde_json::to_string_pretty(&js) {
match response(status)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(body_string(body))
{
Ok(res) => res,
Err(e) => {
error!("can not generate http error response {e}");
internal_error()
}
}
} else {
internal_error()
}
}
}
impl crate::IntoBoxedError for Error {}
pub async fn chconf_from_events_quorum(
q: &PlainEventsQuery,
ctx: &ReqCtx,
@@ -118,7 +296,7 @@ impl ChannelConfigHandler {
node_config: &NodeConfigCached,
) -> Result<StreamResponse, Error> {
let url = req_uri_to_url(req.uri())?;
let q = ChannelConfigQuery::from_url(&url)?;
let q = ChannelConfigQuery::from_url(&url).map_err(|e| Error::BadQuery(e))?;
let conf =
nodenet::channelconfig::channel_config(q.range.clone(), q.channel.clone(), pgqueue, node_config).await?;
match conf {
@@ -176,7 +354,7 @@ impl ChannelConfigsHandler {
async fn channel_configs(&self, req: Requ, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
info!("channel_configs");
let url = req_uri_to_url(req.uri())?;
let q = ChannelConfigQuery::from_url(&url)?;
let q = ChannelConfigQuery::from_url(&url).map_err(|e| Error::BadQuery(e))?;
info!("channel_configs for q {q:?}");
let ch_confs = nodenet::channelconfig::channel_configs(q.channel, ncc).await?;
let ret = response(StatusCode::OK)
@@ -235,7 +413,7 @@ impl ChannelConfigQuorumHandler {
) -> Result<StreamResponse, Error> {
info!("channel_config_quorum");
let url = req_uri_to_url(req.uri())?;
let q = ChannelConfigQuery::from_url(&url)?;
let q = ChannelConfigQuery::from_url(&url).map_err(|e| Error::ChannelConfigQuery(e))?;
info!("channel_config_quorum for q {q:?}");
let ch_confs =
nodenet::configquorum::find_config_basics_quorum(q.channel, q.range.into(), ctx, pgqueue, ncc).await?;
@@ -266,12 +444,12 @@ impl FromUrl for ChannelsWithTypeQuery {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
let s = pairs
.get("scalar_type")
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalar_type"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing scalar_type"))?;
//let scalar_type = ScalarType::from_bsread_str(s)?;
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
let s = pairs
.get("shape")
.ok_or_else(|| Error::with_public_msg_no_trace("missing shape"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing shape"))?;
let shape = Shape::from_dims_str(s)?;
Ok(Self { scalar_type, shape })
}
@@ -302,19 +480,19 @@ impl FromUrl for ScyllaChannelEventSeriesIdQuery {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
let backend = pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing backend"))?
.into();
let name = pairs
.get("channelName")
.ok_or_else(|| Error::with_public_msg_no_trace("missing channelName"))?
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing channelName"))?
.into();
let s = pairs
.get("scalarType")
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalarType"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing scalarType"))?;
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
let s = pairs
.get("shape")
.ok_or_else(|| Error::with_public_msg_no_trace("missing shape"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing shape"))?;
let shape = Shape::from_dims_str(s)?;
let do_create = pairs.get("doCreate").map_or("false", |x| x.as_str()) == "true";
Ok(Self {
@@ -351,15 +529,15 @@ impl FromUrl for ScyllaChannelsActiveQuery {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
let s = pairs
.get("tsedge")
.ok_or_else(|| Error::with_public_msg_no_trace("missing tsedge"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing tsedge"))?;
let tsedge: u64 = s.parse()?;
let s = pairs
.get("shapeKind")
.ok_or_else(|| Error::with_public_msg_no_trace("missing shapeKind"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing shapeKind"))?;
let shape_kind: u32 = s.parse()?;
let s = pairs
.get("scalarType")
.ok_or_else(|| Error::with_public_msg_no_trace("missing scalarType"))?;
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing scalarType"))?;
let scalar_type: ScalarType = serde_json::from_str(&format!("\"{s}\""))?;
info!("parsed scalar type inp: {s:?} val: {scalar_type:?}");
Ok(Self {
@@ -390,7 +568,7 @@ impl ScyllaChannelsActive {
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = req_uri_to_url(req.uri())?;
let q = ScyllaChannelsActiveQuery::from_url(&url)?;
let q = ScyllaChannelsActiveQuery::from_url(&url).map_err(|e| Error::BadQuery(e))?;
let res = self.get_channels(&q, node_config).await?;
let body = ToJsonBody::from(&res).into_body();
Ok(response(StatusCode::OK).body(body)?)
@@ -411,8 +589,10 @@ impl ScyllaChannelsActive {
.node_config
.cluster
.scylla_st()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
let scy = scyllaconn::conn::create_scy_session(scyco).await?;
.ok_or_else(|| Error::ExpectScyllaBackend)?;
let scy = scyllaconn::conn::create_scy_session(scyco)
.await
.map_err(other_err_error)?;
// Database stores tsedge/ts_msp in units of (10 sec), and we additionally map to the grid.
let tsedge = q.tsedge / 10 / (6 * 2) * (6 * 2);
info!(
@@ -427,11 +607,10 @@ impl ScyllaChannelsActive {
"select series from series_by_ts_msp where part = ? and ts_msp = ? and shape_kind = ? and scalar_type = ?",
(part as i32, tsedge as i32, q.shape_kind as i32, q.scalar_type.to_scylla_i32()),
)
.await
.err_conv()?;
.await.map_err(|e| Error::Scylla(e.to_string()))?;
while let Some(row) = res.next().await {
let row = row.err_conv()?;
let (series,): (i64,) = row.into_typed().err_conv()?;
let row = row?;
let (series,): (i64,) = row.into_typed()?;
ret.push(series as u64);
}
}
@@ -456,11 +635,11 @@ impl FromUrl for IocForChannelQuery {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, err::Error> {
let backend = pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing backend"))?
.into();
let name = pairs
.get("channelName")
.ok_or_else(|| Error::with_public_msg_no_trace("missing channelName"))?
.ok_or_else(|| err::Error::with_public_msg_no_trace("missing channelName"))?
.into();
Ok(Self { backend, name })
}
@@ -492,16 +671,13 @@ impl IocForChannel {
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = req_uri_to_url(req.uri())?;
let q = IocForChannelQuery::from_url(&url)?;
let q = IocForChannelQuery::from_url(&url).map_err(|e| Error::BadQuery(e))?;
match self.find(&q, node_config).await {
Ok(k) => {
let body = ToJsonBody::from(&k).into_body();
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => {
let body = body_string(format!("{:?}", e.public_msg()));
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body)?)
}
Err(e) => Ok(e.to_public_response()),
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
@@ -517,7 +693,7 @@ impl IocForChannel {
node_config: &NodeConfigCached,
) -> Result<Option<IocForChannelRes>, Error> {
let dbconf = &node_config.node_config.cluster.database;
let (pg_client, pgjh) = create_connection(dbconf).await?;
let (pg_client, pgjh) = create_connection(dbconf).await.map_err(other_err_error)?;
let rows = pg_client
.query(
"select addr from ioc_by_channel where facility = $1 and channel = $2",
@@ -525,7 +701,7 @@ impl IocForChannel {
)
.await?;
drop(pg_client);
pgjh.await??;
pgjh.await?.map_err(other_err_error)?;
if let Some(row) = rows.first() {
let ioc_addr = row.get(0);
let ret = IocForChannelRes { ioc_addr };
@@ -593,14 +769,13 @@ impl ScyllaSeriesTsMsp {
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = req_uri_to_url(req.uri())?;
let q = ScyllaSeriesTsMspQuery::from_url(&url)?;
let q = ScyllaSeriesTsMspQuery::from_url(&url).map_err(|e| Error::BadQuery(e))?;
match self.get_ts_msps(&q, shared_res).await {
Ok(k) => {
let body = ToJsonBody::from(&k).into_body();
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(body_string(format!("{:?}", e.public_msg())))?),
Err(e) => Ok(e.to_public_response()),
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
@@ -623,12 +798,7 @@ impl ScyllaSeriesTsMsp {
let chconf = shared_res
.pgqueue
.chconf_best_matching_name_range(q.channel.clone(), nano_range)
.await
.map_err(|e| Error::with_msg_no_trace(format!("error from pg worker: {e}")))?
.recv()
.await
.unwrap()
.unwrap();
.await??;
use scyllaconn::SeriesId;
let sid = SeriesId::new(chconf.series());
let scyqueue = shared_res.scyqueue.clone().unwrap();
@@ -710,8 +880,7 @@ impl AmbigiousChannelNames {
let body = ToJsonBody::from(&k).into_body();
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(body_string(format!("{:?}", e.public_msg())))?),
Err(e) => Ok(e.to_public_response()),
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
@@ -723,7 +892,7 @@ impl AmbigiousChannelNames {
async fn process(&self, ncc: &NodeConfigCached) -> Result<AmbigiousChannelNamesResponse, Error> {
let dbconf = &ncc.node_config.cluster.database;
let (pg_client, pgjh) = create_connection(dbconf).await?;
let (pg_client, pgjh) = create_connection(dbconf).await.map_err(other_err_error)?;
let rows = pg_client
.query(
"select t2.series, t2.channel, t2.scalar_type, t2.shape_dims, t2.agg_kind from series_by_channel t1, series_by_channel t2 where t2.channel = t1.channel and t2.series != t1.series",
@@ -731,14 +900,14 @@ impl AmbigiousChannelNames {
)
.await?;
drop(pg_client);
pgjh.await??;
pgjh.await?.map_err(other_err_error)?;
let mut ret = AmbigiousChannelNamesResponse { ambigious: Vec::new() };
for row in rows {
let g = AmbigiousChannel {
series: row.get::<_, i64>(0) as u64,
name: row.get(1),
scalar_type: ScalarType::from_scylla_i32(row.get(2))?,
shape: Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(3))?,
scalar_type: ScalarType::from_scylla_i32(row.get(2)).map_err(other_err_error)?,
shape: Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(3)).map_err(other_err_error)?,
};
ret.ambigious.push(g);
}
@@ -798,8 +967,7 @@ impl GenerateScyllaTestData {
let body = ToJsonBody::from(&k).into_body();
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(body_string(format!("{:?}", e.public_msg())))?),
Err(e) => Ok(e.to_public_response()),
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(body_empty())?)
@@ -811,25 +979,24 @@ impl GenerateScyllaTestData {
async fn process(&self, node_config: &NodeConfigCached) -> Result<(), Error> {
let scyconf = node_config.node_config.cluster.scylla_st().unwrap();
let scy = scyllaconn::conn::create_scy_session(scyconf).await?;
let scy = scyllaconn::conn::create_scy_session(scyconf)
.await
.map_err(other_err_error)?;
let series: u64 = 42001;
// TODO query `ts_msp` for all MSP values und use that to delete from event table first.
// Only later delete also from the `ts_msp` table.
let it = scy
.query_iter("select ts_msp from ts_msp where series = ?", (series as i64,))
.await
.err_conv()?;
.await?;
let mut it = it.into_typed::<(i64,)>();
while let Some(row) = it.next().await {
let row = row.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let row = row?;
let values = (series as i64, row.0);
scy.query("delete from events_scalar_f64 where series = ? and ts_msp = ?", values)
.await
.err_conv()?;
.await?;
}
scy.query("delete from ts_msp where series = ?", (series as i64,))
.await
.err_conv()?;
.await?;
// Generate
let (msps, lsps, pulses, vals) = test_data_f64_01();
@@ -840,8 +1007,7 @@ impl GenerateScyllaTestData {
"insert into ts_msp (series, ts_msp) values (?, ?)",
(series as i64, msp as i64),
)
.await
.err_conv()?;
.await?;
}
last = msp;
}
@@ -850,8 +1016,7 @@ impl GenerateScyllaTestData {
"insert into events_scalar_f64 (series, ts_msp, ts_lsp, pulse, value) values (?, ?, ?, ?, ?)",
(series as i64, msp as i64, lsp as i64, pulse as i64, val),
)
.await
.err_conv()?;
.await?;
}
Ok(())
}

4
crates/httpret/src/err.rs
View File

@@ -1,7 +1,6 @@
use err::ToPublicError;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsVal;
use std::fmt;
use taskrun::tokio;
@@ -108,3 +107,6 @@ impl Convable for std::array::TryFromSliceError {}
impl Convable for err::anyhow::Error {}
impl Convable for crate::RetrievalError {}
impl Convable for httpclient::Error {}
impl Convable for netpod::UriError {}
impl Convable for nodenet::configquorum::Error {}
impl Convable for nodenet::channelconfig::Error {}

2
crates/httpret/src/httpret.rs
View File

@@ -87,6 +87,8 @@ impl IntoBoxedError for api4::databuffer_tools::FindActiveError {}
impl IntoBoxedError for std::string::FromUtf8Error {}
impl IntoBoxedError for std::io::Error {}
impl IntoBoxedError for dbconn::worker::Error {}
impl IntoBoxedError for netpod::UriError {}
impl IntoBoxedError for crate::api4::binned::Error {}
impl<E> From<E> for RetrievalError
where

6
crates/httpret/src/proxy.rs
View File

@@ -4,7 +4,6 @@ pub mod api4;
use crate::api1::channel_search_configs_v1;
use crate::api1::channel_search_list_v1;
use crate::api1::gather_json_2_v1;
use crate::bodystream::response_err_msg;
use crate::err::Error;
use crate::gather::gather_get_json_generic;
use crate::gather::SubRes;
@@ -21,6 +20,7 @@ use http::StatusCode;
use httpclient::body_empty;
use httpclient::body_stream;
use httpclient::body_string;
use httpclient::error_response;
use httpclient::http;
use httpclient::http::header;
use httpclient::read_body_bytes;
@@ -509,9 +509,9 @@ where
let mut query = match QT::from_url(&url) {
Ok(k) => k,
Err(_) => {
let msg = format!("malformed request or missing parameters {:?}", req.uri());
let msg = format!("malformed request or missing parameters {}", req.uri());
warn!("{msg}");
return Ok(response_err_msg(StatusCode::BAD_REQUEST, msg)?);
return Ok(error_response(msg, ctx.reqid()));
}
};
trace!("proxy_backend_query {:?} {:?}", query, req.uri());

15
crates/httpret/src/proxy/api4/events.rs
View File

@@ -8,6 +8,7 @@ use crate::ReqCtx;
use http::header;
use http::Method;
use http::Request;
use http::Response;
use http::StatusCode;
use http::Uri;
use httpclient::body_empty;
@@ -68,7 +69,7 @@ impl EventsHandler {
let url = req_uri_to_url(&head.uri)?;
let pairs = get_url_query_pairs(&url);
let evq = PlainEventsQuery::from_pairs(&pairs)?;
debug!("{:?}", evq);
debug!("handle_framed {evq:?}");
let query_host = get_query_host_for_backend(evq.backend(), proxy_config)?;
let url_str = format!(
"{}{}",
@@ -92,10 +93,14 @@ impl EventsHandler {
let (head, body) = res.into_parts();
if head.status != StatusCode::OK {
warn!("backend returned error: {head:?}");
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body_empty())?)
} else {
debug!("backend returned OK");
Ok(response(StatusCode::OK).body(body_stream(StreamIncoming::new(body)))?)
}
let mut resb = Response::builder().status(head.status);
for h in head.headers {
if let (Some(hn), hv) = h {
resb = resb.header(hn, hv);
}
}
let res = resb.body(body_stream(StreamIncoming::new(body)))?;
Ok(res)
}
}