daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor usage of common error type more

Browse Source
This commit is contained in:
Dominik Werder
2021-12-08 13:20:07 +01:00
parent c39af81097
commit 3c64eafd14
56 changed files with 751 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
httpret/src/api1.rs
View File

@@ -1,10 +1,10 @@
use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::{response, BodyStream};
use bytes::{BufMut, BytesMut};
use disk::eventchunker::{EventChunkerConf, EventFull};
use err::Error;
use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use futures_util::{FutureExt, StreamExt, TryFutureExt, TryStreamExt};
use http::{Method, StatusCode};
use hyper::{Body, Client, Request, Response};
use items::{RangeCompletableItem, Sitemty, StreamItem};
@@ -736,7 +736,7 @@ impl Stream for DataApiPython3DataStream {
}
});
//let _ = Box::new(s) as Box<dyn Stream<Item = Result<BytesMut, Error>> + Unpin>;
self.chan_stream = Some(Box::pin(s));
self.chan_stream = Some(Box::pin(s.map_err(Error::from)));
continue;
}
Ready(Err(e)) => {
@@ -754,10 +754,9 @@ impl Stream for DataApiPython3DataStream {
} else {
let channel = self.channels[self.chan_ix].clone();
self.chan_ix += 1;
self.config_fut = Some(Box::pin(read_local_config(
channel.clone(),
self.node_config.node.clone(),
)));
self.config_fut = Some(Box::pin(
read_local_config(channel.clone(), self.node_config.node.clone()).map_err(Error::from),
));
continue;
}
}

10
httpret/src/channelarchiver.rs
View File

@@ -1,7 +1,7 @@
use crate::err::Error;
use crate::response;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use futures_util::{StreamExt, TryStreamExt};
use http::{header, Method, Request, Response, StatusCode};
use hyper::Body;
use netpod::log::*;
@@ -112,6 +112,7 @@ impl ScanIndexFiles {
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_index_files(conf.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
@@ -151,6 +152,7 @@ impl ScanChannels {
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_channels(conf.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
@@ -191,6 +193,7 @@ impl ChannelNames {
))?;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
@@ -232,6 +235,7 @@ impl ScanConfigs {
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = archeng::configs::ConfigStream::new(stream, conf.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
@@ -294,6 +298,7 @@ impl BlockRefStream {
}
Err(e) => Err(e),
});
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),
@@ -365,6 +370,7 @@ impl BlockStream {
}
Err(e) => Err(e),
});
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),

72
httpret/src/err.rs Normal file
View File

@@ -0,0 +1,72 @@
use std::fmt;
#[derive(Debug)]
pub struct Error(::err::Error);
impl Error {
pub fn with_msg<S: Into<String>>(s: S) -> Self {
Self(::err::Error::with_msg(s))
}
pub fn with_msg_no_trace<S: Into<String>>(s: S) -> Self {
Self(::err::Error::with_msg_no_trace(s))
}
pub fn msg(&self) -> &str {
self.0.msg()
}
pub fn reason(&self) -> Option<::err::Reason> {
self.0.reason()
}
pub fn public_msg(&self) -> Option<&Vec<String>> {
self.0.public_msg()
}
}
impl fmt::Display for Error {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(self, fmt)
}
}
impl std::error::Error for Error {}
impl From<::err::Error> for Error {
fn from(x: ::err::Error) -> Self {
Self(x)
}
}
impl From<Error> for ::err::Error {
fn from(x: Error) -> Self {
x.0
}
}
pub trait Convable {}
impl<T: Convable> From<T> for Error
where
T: ToString,
{
fn from(x: T) -> Self {
Self(::err::Error::from_string(x))
}
}
impl Convable for std::net::AddrParseError {}
impl Convable for std::string::FromUtf8Error {}
impl Convable for fmt::Error {}
impl Convable for std::io::Error {}
impl Convable for std::num::ParseIntError {}
impl Convable for dbconn::pg::Error {}
impl Convable for tokio::task::JoinError {}
impl Convable for tokio::time::error::Elapsed {}
impl Convable for serde_json::Error {}
impl Convable for chrono::ParseError {}
impl Convable for url::ParseError {}
impl Convable for http::uri::InvalidUri {}
impl Convable for http::Error {}
impl Convable for hyper::Error {}

2
httpret/src/gather.rs
View File

@@ -1,5 +1,5 @@
use crate::err::Error;
use crate::response;
use err::Error;
use futures_util::{select, FutureExt};
use http::{Method, StatusCode};
use hyper::{Body, Client, Request, Response};

50
httpret/src/httpret.rs
View File

@@ -1,19 +1,20 @@
pub mod api1;
pub mod channelarchiver;
pub mod err;
pub mod gather;
pub mod proxy;
pub mod pulsemap;
pub mod search;
use crate::err::Error;
use crate::gather::gather_get_json;
use crate::pulsemap::UpdateTask;
use bytes::Bytes;
use disk::binned::query::PreBinnedQuery;
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
use err::Error;
use future::Future;
use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use futures_util::{FutureExt, StreamExt, TryStreamExt};
use http::{HeaderMap, Method, StatusCode};
use hyper::server::conn::AddrStream;
use hyper::service::{make_service_fn, service_fn};
@@ -89,7 +90,7 @@ where
{
type Output = <F as Future>::Output;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
let h = std::panic::catch_unwind(AssertUnwindSafe(|| self.f.poll_unpin(cx)));
match h {
Ok(k) => k,
@@ -101,7 +102,7 @@ where
}
None => {}
}
Poll::Ready(Err(Error::from(format!("{:?}", e))))
Poll::Ready(Err(Error::with_msg(format!("{:?}", e))))
}
}
}
@@ -369,7 +370,7 @@ struct BodyStreamWrap(netpod::BodyStream);
impl hyper::body::HttpBody for BodyStreamWrap {
type Data = bytes::Bytes;
type Error = Error;
type Error = ::err::Error;
fn poll_data(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Self::Data, Self::Error>>> {
self.0.inner.poll_next_unpin(cx)
@@ -417,7 +418,7 @@ where
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
Ready(Some(Err(e))) => {
error!("body stream error: {:?}", e);
Ready(Some(Err(e.into())))
Ready(Some(Err(Error::from(e))))
}
Ready(None) => Ready(None),
Pending => Pending,
@@ -439,8 +440,23 @@ trait ToPublicResponse {
impl ToPublicResponse for Error {
fn to_public_response(&self) -> Response<Body> {
let status = match self.reason() {
Some(err::Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(err::Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
Some(::err::Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(::err::Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
let msg = match self.public_msg() {
Some(v) => v.join("\n"),
_ => String::new(),
};
response(status).body(Body::from(msg)).unwrap()
}
}
impl ToPublicResponse for ::err::Error {
fn to_public_response(&self) -> Response<Body> {
let status = match self.reason() {
Some(::err::Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(::err::Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
let msg = match self.public_msg() {
@@ -476,7 +492,9 @@ async fn binned_inner(req: Request<Body>, node_config: &NodeConfigCached) -> Res
async fn binned_binary(query: BinnedQuery, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_bytes_for_http(&query, node_config).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, format!("binned_binary")))?,
Ok(s) => {
response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_binary")))?
}
Err(e) => {
if query.report_error() {
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?
@@ -491,7 +509,7 @@ async fn binned_binary(query: BinnedQuery, node_config: &NodeConfigCached) -> Re
async fn binned_json(query: BinnedQuery, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_json(&query, node_config).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, format!("binned_json")))?,
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_json")))?,
Err(e) => {
if query.report_error() {
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?
@@ -525,7 +543,7 @@ async fn prebinned_inner(req: Request<Body>, node_config: &NodeConfigCached) ->
});
let fut = disk::binned::prebinned::pre_binned_bytes_for_http(node_config, &query).instrument(span1);
let ret = match fut.await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, desc))?,
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), desc))?,
Err(e) => {
if query.report_error() {
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?
@@ -574,7 +592,10 @@ async fn plain_events_binary(req: Request<Body>, node_config: &NodeConfigCached)
);
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), AggKind::Plain, node_config).await?;
let s = s.map(|item| item.make_frame());
let ret = response(StatusCode::OK).body(BodyStream::wrapped(s, format!("plain_events_binary")))?;
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
s.map_err(Error::from),
format!("plain_events_binary"),
))?;
Ok(ret)
}
@@ -591,7 +612,10 @@ async fn plain_events_json(req: Request<Body>, node_config: &NodeConfigCached) -
query.do_log(),
);
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), AggKind::Plain, node_config).await?;
let ret = response(StatusCode::OK).body(BodyStream::wrapped(s, format!("plain_events_json")))?;
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
s.map_err(Error::from),
format!("plain_events_json"),
))?;
Ok(ret)
}

2
httpret/src/proxy.rs
View File

@@ -1,10 +1,10 @@
pub mod api4;
use crate::api1::{channel_search_configs_v1, channel_search_list_v1, gather_json_2_v1, proxy_distribute_v1};
use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::{api_1_docs, api_4_docs, response, Cont};
use disk::events::PlainEventsJsonQuery;
use err::Error;
use futures_core::Stream;
use futures_util::pin_mut;
use http::{Method, StatusCode};

9
httpret/src/proxy/api4.rs
View File

@@ -1,6 +1,6 @@
use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::response;
use err::Error;
use futures_core::Future;
use http::{header, Request, Response, StatusCode};
use hyper::Body;
@@ -68,7 +68,8 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
let res = ChannelSearchResult { channels: res };
let res = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&res)?))?;
.body(Body::from(serde_json::to_string(&res)?))
.map_err(Error::from)?;
Ok(res)
};
let ret = gather_get_json_generic(
@@ -83,6 +84,8 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
.await?;
Ok(ret)
} else {
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::from(format!("{:?}", proxy_config.name)))?)
Ok(response(StatusCode::NOT_ACCEPTABLE)
.body(Body::from(format!("{:?}", proxy_config.name)))
.map_err(Error::from)?)
}
}

17
httpret/src/pulsemap.rs
View File

@@ -1,11 +1,9 @@
use crate::err::Error;
use crate::response;
use bytes::Buf;
use bytes::BytesMut;
use err::Error;
use bytes::{Buf, BytesMut};
use futures_util::stream::FuturesOrdered;
use futures_util::FutureExt;
use http::Uri;
use http::{Method, StatusCode};
use http::{Method, StatusCode, Uri};
use hyper::{Body, Request, Response};
use netpod::log::*;
use netpod::NodeConfigCached;
@@ -17,14 +15,11 @@ use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use std::time::Instant;
use std::time::{Duration, Instant};
use std::{io::SeekFrom, path::PathBuf};
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio::task::JoinHandle;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncSeekExt},
};
pub struct MapPulseHisto {
_pulse: u64,

2
httpret/src/search.rs
View File

@@ -1,5 +1,5 @@
use crate::err::Error;
use crate::response;
use err::Error;
use http::header;
use hyper::{Body, Request, Response, StatusCode};
use netpod::log::*;