daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring

Browse Source
This commit is contained in:
Dominik Werder
2022-06-15 14:27:38 +02:00
parent 7063842c4c
commit 9e3395bf13
25 changed files with 429 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
httpret/src/api1.rs
View File

@@ -534,6 +534,7 @@ pub struct DataApiPython3DataStream {
chan_stream: Option<Pin<Box<dyn Stream<Item = Result<BytesMut, Error>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<Config, Error>> + Send>>>,
disk_io_tune: DiskIoTune,
#[allow(unused)]
do_decompress: bool,
#[allow(unused)]
event_count: u64,
@@ -730,13 +731,7 @@ impl Stream for DataApiPython3DataStream {
};
let channel = self.channels[self.chan_ix - 1].clone();
debug!("found channel_config for {}: {:?}", channel.name, entry);
let evq = RawEventsQuery {
channel,
range: self.range.clone(),
agg_kind: netpod::AggKind::EventBlobs,
disk_io_tune: self.disk_io_tune.clone(),
do_decompress: self.do_decompress,
};
let evq = RawEventsQuery::new(channel, self.range.clone(), netpod::AggKind::EventBlobs);
let perf_opts = PerfOpts { inmem_bufcap: 1024 * 4 };
// TODO is this a good to place decide this?
let s = if self.node_config.node_config.cluster.is_central_storage {

135
httpret/src/bodystream.rs Normal file
View File

@@ -0,0 +1,135 @@
use crate::err::Error;
use bytes::Bytes;
use futures_core::Stream;
use futures_util::StreamExt;
use http::HeaderMap;
use http::{Response, StatusCode};
use hyper::Body;
use netpod::log::*;
use netpod::APP_JSON;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::task::{Context, Poll};
use tracing::field::Empty;
use tracing::{span, Level};
fn proxy_mark() -> &'static str {
"7c5e408a"
}
pub fn response<T>(status: T) -> http::response::Builder
where
http::StatusCode: std::convert::TryFrom<T>,
<http::StatusCode as std::convert::TryFrom<T>>::Error: Into<http::Error>,
{
Response::builder()
.status(status)
.header("Access-Control-Allow-Origin", "*")
.header("Access-Control-Allow-Headers", "*")
.header("x-proxy-log-mark", proxy_mark())
}
pub struct BodyStream<S> {
inp: S,
desc: String,
}
impl<S, I> BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin + Send + 'static,
I: Into<Bytes> + Sized + 'static,
{
pub fn new(inp: S, desc: String) -> Self {
Self { inp, desc }
}
pub fn wrapped(inp: S, desc: String) -> Body {
Body::wrap_stream(Self::new(inp, desc))
}
}
impl<S, I> Stream for BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin,
I: Into<Bytes> + Sized,
{
type Item = Result<I, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let span1 = span!(Level::INFO, "httpret::BodyStream", desc = Empty);
span1.record("desc", &self.desc.as_str());
span1.in_scope(|| {
use Poll::*;
let t = std::panic::catch_unwind(AssertUnwindSafe(|| self.inp.poll_next_unpin(cx)));
match t {
Ok(r) => match r {
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
Ready(Some(Err(e))) => {
error!("body stream error: {e:?}");
Ready(Some(Err(Error::from(e))))
}
Ready(None) => Ready(None),
Pending => Pending,
},
Err(e) => {
error!("panic caught in httpret::BodyStream: {e:?}");
let e = Error::with_msg(format!("panic caught in httpret::BodyStream: {e:?}"));
Ready(Some(Err(e)))
}
}
})
}
}
pub trait ToPublicResponse {
fn to_public_response(&self) -> Response<Body>;
}
impl ToPublicResponse for Error {
fn to_public_response(&self) -> Response<Body> {
self.0.to_public_response()
}
}
impl ToPublicResponse for ::err::Error {
fn to_public_response(&self) -> Response<Body> {
use err::Reason;
let e = self.to_public_error();
let status = match e.reason() {
Some(Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
let msg = match serde_json::to_string(&e) {
Ok(s) => s,
Err(_) => "can not serialize error".into(),
};
match response(status)
.header(http::header::ACCEPT, APP_JSON)
.body(Body::from(msg))
{
Ok(res) => res,
Err(e) => {
error!("can not generate http error response {e:?}");
let mut res = Response::new(Body::default());
*res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
res
}
}
}
}
struct BodyStreamWrap(netpod::BodyStream);
impl hyper::body::HttpBody for BodyStreamWrap {
type Data = bytes::Bytes;
type Error = ::err::Error;
fn poll_data(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Self::Data, Self::Error>>> {
self.0.inner.poll_next_unpin(cx)
}
fn poll_trailers(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<Option<HeaderMap>, Self::Error>> {
Poll::Ready(Ok(None))
}
}

69
httpret/src/channelconfig.rs
View File

@@ -2,7 +2,7 @@ use crate::err::Error;
use crate::{response, ToPublicResponse};
use dbconn::create_connection;
use disk::binned::query::PreBinnedQuery;
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
use disk::events::PlainEventsQuery;
use http::{Method, Request, Response, StatusCode};
use hyper::Body;
use netpod::log::*;
@@ -24,11 +24,11 @@ pub struct ChConf {
pub shape: Shape,
}
pub async fn chconf_from_events_binary(_q: &PlainEventsBinaryQuery, _conf: &NodeConfigCached) -> Result<ChConf, Error> {
pub async fn chconf_from_events_binary(_q: &PlainEventsQuery, _conf: &NodeConfigCached) -> Result<ChConf, Error> {
err::todoval()
}
pub async fn chconf_from_events_json(q: &PlainEventsJsonQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
pub async fn chconf_from_events_json(q: &PlainEventsQuery, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
if q.channel().backend != ncc.node_config.cluster.backend {
warn!(
"Mismatched backend {} VS {}",
@@ -843,36 +843,37 @@ impl ChannelFromSeries {
#[derive(Clone, Debug, Deserialize)]
pub struct IocForChannelQuery {
channel: String,
facility: String,
#[serde(rename = "channelName")]
channel_name: String,
}
impl FromUrl for IocForChannelQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
let pairs = get_url_query_pairs(url);
let channel = pairs
let facility = pairs
.get("facility")
.ok_or_else(|| Error::with_public_msg_no_trace("missing facility"))?
.into();
let channel_name = pairs
.get("channelName")
.ok_or_else(|| Error::with_public_msg_no_trace("missing channelName"))?
.into();
Ok(Self { channel })
Ok(Self { facility, channel_name })
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ChannelIoc {
channel: String,
ioc: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct IocForChannelRes {
channels: Vec<Channel>,
#[serde(rename = "iocAddr")]
ioc_addr: String,
}
pub struct IocForChannel {}
impl IocForChannel {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/ioc/channel" {
if req.uri().path() == "/api/4/channel/ioc" {
Some(Self {})
} else {
None
@@ -889,9 +890,16 @@ impl IocForChannel {
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = IocForChannelQuery::from_url(&url)?;
let res = self.find(&q, node_config).await?;
let body = Body::from(serde_json::to_vec(&res)?);
Ok(response(StatusCode::OK).body(body)?)
match self.find(&q, node_config).await {
Ok(k) => {
let body = Body::from(serde_json::to_vec(&k)?);
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => {
let body = Body::from(format!("{:?}", e.public_msg()));
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body)?)
}
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
@@ -900,13 +908,26 @@ impl IocForChannel {
}
}
async fn find(&self, q: &IocForChannelQuery, node_config: &NodeConfigCached) -> Result<IocForChannelRes, Error> {
// TODO implement lookup in postgres
let _ = q;
let _pgconn = create_connection(&node_config.node_config.cluster.database).await?;
let _facility = "scylla";
let ret = IocForChannelRes { channels: vec![] };
Ok(ret)
async fn find(
&self,
q: &IocForChannelQuery,
node_config: &NodeConfigCached,
) -> Result<Option<IocForChannelRes>, Error> {
let dbconf = &node_config.node_config.cluster.database;
let pg_client = create_connection(dbconf).await?;
let rows = pg_client
.query(
"select addr from ioc_by_channel where facility = $1 and channel = $2",
&[&q.facility, &q.channel_name],
)
.await?;
if let Some(row) = rows.first() {
let ioc_addr = row.get(0);
let ret = IocForChannelRes { ioc_addr };
Ok(Some(ret))
} else {
Ok(None)
}
}
}

16
httpret/src/events.rs
View File

@@ -1,7 +1,7 @@
use crate::channelconfig::{chconf_from_events_binary, chconf_from_events_json};
use crate::err::Error;
use crate::{response, response_err, BodyStream, ToPublicResponse};
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
use disk::events::PlainEventsQuery;
use futures_util::{StreamExt, TryStreamExt};
use http::{Method, Request, Response, StatusCode};
use hyper::Body;
@@ -52,14 +52,9 @@ async fn plain_events(req: Request<Body>, node_config: &NodeConfigCached) -> Res
async fn plain_events_binary(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
debug!("httpret plain_events_binary req: {:?}", req);
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let query = PlainEventsBinaryQuery::from_url(&url)?;
let query = PlainEventsQuery::from_url(&url)?;
let chconf = chconf_from_events_binary(&query, node_config).await?;
let op = disk::channelexec::PlainEvents::new(
query.channel().clone(),
query.range().clone(),
query.disk_io_buffer_size(),
node_config.clone(),
);
let op = disk::channelexec::PlainEvents::new(query.channel().clone(), query.range().clone(), node_config.clone());
let s = disk::channelexec::channel_exec(
op,
query.channel(),
@@ -81,12 +76,13 @@ async fn plain_events_binary(req: Request<Body>, node_config: &NodeConfigCached)
async fn plain_events_json(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!("httpret plain_events_json req: {:?}", req);
let (head, _body) = req.into_parts();
let query = PlainEventsJsonQuery::from_request_head(&head)?;
let query = PlainEventsQuery::from_request_head(&head)?;
let chconf = chconf_from_events_json(&query, node_config).await?;
let op = disk::channelexec::PlainEventsJson::new(
// TODO pass only the query, not channel, range again:
query.clone(),
query.channel().clone(),
query.range().clone(),
query.disk_io_buffer_size(),
query.timeout(),
node_config.clone(),
query.events_max().unwrap_or(u64::MAX),

22
httpret/src/evinfo.rs
View File

@@ -9,7 +9,7 @@ use disk::decode::Endianness;
use disk::decode::EventValueFromBytes;
use disk::decode::EventValueShape;
use disk::decode::NumFromBytes;
use disk::events::PlainEventsJsonQuery;
use disk::events::PlainEventsQuery;
use disk::merge::mergedfromremotes::MergedFromRemotes;
use futures_util::FutureExt;
use futures_util::Stream;
@@ -58,7 +58,7 @@ impl EventInfoScan {
}
let (head, _body) = req.into_parts();
let url = Url::parse(&format!("dummy:{}", head.uri))?;
let query = PlainEventsJsonQuery::from_url(&url)?;
let query = PlainEventsQuery::from_url(&url)?;
let ret = match Self::exec(&query, node_config).await {
Ok(stream) => {
//
@@ -71,7 +71,7 @@ impl EventInfoScan {
}
pub async fn exec(
query: &PlainEventsJsonQuery,
query: &PlainEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>, Error> {
let chconf = chconf_from_events_json(&query, node_config).await?;
@@ -95,14 +95,14 @@ impl EventInfoScan {
}
pub struct EvInfoFunc {
query: PlainEventsJsonQuery,
query: PlainEventsQuery,
timeout: Duration,
node_config: NodeConfigCached,
events_max: u64,
}
impl EvInfoFunc {
pub fn new(query: PlainEventsJsonQuery, timeout: Duration, events_max: u64, node_config: NodeConfigCached) -> Self {
pub fn new(query: PlainEventsQuery, timeout: Duration, events_max: u64, node_config: NodeConfigCached) -> Self {
Self {
query,
timeout,
@@ -149,16 +149,8 @@ impl ChannelExecFunction for EvInfoFunc {
let _ = byte_order;
let _ = event_value_shape;
let perf_opts = PerfOpts { inmem_bufcap: 4096 };
// TODO let PlainEventsJsonQuery provide the tune
let mut disk_io_tune = netpod::DiskIoTune::default();
disk_io_tune.read_buffer_len = self.query.disk_io_buffer_size();
let evq = RawEventsQuery {
channel: self.query.channel().clone(),
range: self.query.range().clone(),
agg_kind: AggKind::Plain,
disk_io_tune,
do_decompress: true,
};
// TODO let PlainEventsJsonQuery provide the tune and pass to RawEventsQuery:
let evq = RawEventsQuery::new(self.query.channel().clone(), self.query.range().clone(), AggKind::Plain);
// TODO Use a Merged-From-Multiple-Local-Splits.
// TODO Pass the read buffer size from query parameter: GPFS needs a larger buffer..

173
httpret/src/httpret.rs
View File

@@ -1,4 +1,5 @@
pub mod api1;
pub mod bodystream;
pub mod channelarchiver;
pub mod channelconfig;
pub mod download;
@@ -11,16 +12,16 @@ pub mod pulsemap;
pub mod search;
pub mod settings;
use self::bodystream::{BodyStream, ToPublicResponse};
use crate::bodystream::response;
use crate::err::Error;
use crate::gather::gather_get_json;
use crate::pulsemap::UpdateTask;
use bytes::Bytes;
use channelconfig::{chconf_from_binned, ChConf};
use disk::binned::query::PreBinnedQuery;
use future::Future;
use futures_core::Stream;
use futures_util::{FutureExt, StreamExt, TryStreamExt};
use http::{HeaderMap, Method, StatusCode};
use http::{Method, StatusCode};
use hyper::server::conn::AddrStream;
use hyper::service::{make_service_fn, service_fn};
use hyper::{server::Server, Body, Request, Response};
@@ -41,14 +42,9 @@ use std::sync::{Once, RwLock, RwLockWriteGuard};
use std::time::SystemTime;
use std::{future, net, panic, pin, task};
use task::{Context, Poll};
use tracing::field::Empty;
use tracing::Instrument;
use url::Url;
fn proxy_mark() -> &'static str {
"7c5e408a"
}
pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
static STATUS_BOARD_INIT: Once = Once::new();
STATUS_BOARD_INIT.call_once(|| {
@@ -390,123 +386,6 @@ pub fn api_1_docs(path: &str) -> Result<Response<Body>, Error> {
Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
}
fn response<T>(status: T) -> http::response::Builder
where
http::StatusCode: std::convert::TryFrom<T>,
<http::StatusCode as std::convert::TryFrom<T>>::Error: Into<http::Error>,
{
Response::builder()
.status(status)
.header("Access-Control-Allow-Origin", "*")
.header("Access-Control-Allow-Headers", "*")
.header("x-proxy-log-mark", proxy_mark())
}
struct BodyStreamWrap(netpod::BodyStream);
impl hyper::body::HttpBody for BodyStreamWrap {
type Data = bytes::Bytes;
type Error = ::err::Error;
fn poll_data(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Self::Data, Self::Error>>> {
self.0.inner.poll_next_unpin(cx)
}
fn poll_trailers(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<Option<HeaderMap>, Self::Error>> {
Poll::Ready(Ok(None))
}
}
struct BodyStream<S> {
inp: S,
desc: String,
}
impl<S, I> BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin + Send + 'static,
I: Into<Bytes> + Sized + 'static,
{
pub fn new(inp: S, desc: String) -> Self {
Self { inp, desc }
}
pub fn wrapped(inp: S, desc: String) -> Body {
Body::wrap_stream(Self::new(inp, desc))
}
}
impl<S, I> Stream for BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin,
I: Into<Bytes> + Sized,
{
type Item = Result<I, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let span1 = span!(Level::INFO, "httpret::BodyStream", desc = Empty);
span1.record("desc", &self.desc.as_str());
span1.in_scope(|| {
use Poll::*;
let t = std::panic::catch_unwind(AssertUnwindSafe(|| self.inp.poll_next_unpin(cx)));
match t {
Ok(r) => match r {
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
Ready(Some(Err(e))) => {
error!("body stream error: {e:?}");
Ready(Some(Err(Error::from(e))))
}
Ready(None) => Ready(None),
Pending => Pending,
},
Err(e) => {
error!("panic caught in httpret::BodyStream: {e:?}");
let e = Error::with_msg(format!("panic caught in httpret::BodyStream: {e:?}"));
Ready(Some(Err(e)))
}
}
})
}
}
trait ToPublicResponse {
fn to_public_response(&self) -> Response<Body>;
}
impl ToPublicResponse for Error {
fn to_public_response(&self) -> Response<Body> {
self.0.to_public_response()
}
}
impl ToPublicResponse for ::err::Error {
fn to_public_response(&self) -> Response<Body> {
use ::err::Reason;
let e = self.to_public_error();
let status = match e.reason() {
Some(Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
let msg = match serde_json::to_string(&e) {
Ok(s) => s,
Err(_) => "can not serialize error".into(),
};
match response(status)
.header(http::header::ACCEPT, APP_JSON)
.body(Body::from(msg))
{
Ok(res) => res,
Err(e) => {
error!("can not generate http error response {e:?}");
let mut res = Response::new(Body::default());
*res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
res
}
}
}
}
pub struct StatusBoardAllHandler {}
impl StatusBoardAllHandler {
@@ -530,7 +409,10 @@ impl StatusBoardAllHandler {
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
match binned_inner(req, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => Ok(e.to_public_response()),
Err(e) => {
error!("fn binned: {e:?}");
Ok(e.to_public_response())
}
}
}
@@ -556,20 +438,13 @@ async fn binned_binary(
chconf: ChConf,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_bytes_for_http(&query, chconf.scalar_type, chconf.shape, node_config).await {
Ok(s) => {
response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_binary")))?
}
Err(e) => {
if query.report_error() {
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?
} else {
error!("fn binned_binary: {:?}", e);
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?
}
}
};
Ok(ret)
let body_stream =
disk::binned::binned_bytes_for_http(&query, chconf.scalar_type, chconf.shape, node_config).await?;
let res = response(StatusCode::OK).body(BodyStream::wrapped(
body_stream.map_err(Error::from),
format!("binned_binary"),
))?;
Ok(res)
}
async fn binned_json(
@@ -577,18 +452,12 @@ async fn binned_json(
chconf: ChConf,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let ret = match disk::binned::binned_json(&query, chconf.scalar_type, chconf.shape, node_config).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s.map_err(Error::from), format!("binned_json")))?,
Err(e) => {
if query.report_error() {
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?
} else {
error!("fn binned_json: {:?}", e);
response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?
}
}
};
Ok(ret)
let body_stream = disk::binned::binned_json(&query, chconf.scalar_type, chconf.shape, node_config).await?;
let res = response(StatusCode::OK).body(BodyStream::wrapped(
body_stream.map_err(Error::from),
format!("binned_json"),
))?;
Ok(res)
}
async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {

4
httpret/src/proxy.rs
View File

@@ -5,7 +5,7 @@ use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::pulsemap::MapPulseQuery;
use crate::{api_1_docs, api_4_docs, response, response_err, Cont};
use disk::events::PlainEventsJsonQuery;
use disk::events::PlainEventsQuery;
use futures_core::Stream;
use futures_util::pin_mut;
use http::{Method, StatusCode};
@@ -108,7 +108,7 @@ async fn proxy_http_service_try(req: Request<Body>, proxy_config: &ProxyConfig)
} else if path == "/api/4/search/channel" {
Ok(api4::channel_search(req, proxy_config).await?)
} else if path == "/api/4/events" {
Ok(proxy_single_backend_query::<PlainEventsJsonQuery>(req, proxy_config).await?)
Ok(proxy_single_backend_query::<PlainEventsQuery>(req, proxy_config).await?)
} else if path.starts_with("/api/4/map/pulse/") {
Ok(proxy_single_backend_query::<MapPulseQuery>(req, proxy_config).await?)
} else if path == "/api/4/binned" {