daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optionally limit the number of retrieved events

Browse Source
This commit is contained in:
Dominik Werder
2022-02-22 21:32:55 +01:00
parent a0bc67d3a3
commit 36ecc858fd
9 changed files with 65 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
httpret/src/events.rs
View File

@@ -24,23 +24,15 @@ impl EventsHandler {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
let ret = match plain_events(req, node_config).await {
Ok(res) => res,
Err(e) => response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?,
};
Ok(ret)
match plain_events(req, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => Ok(e.to_public_response()),
}
}
}
async fn plain_events(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
match plain_events_inner(req, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => Ok(e.to_public_response()),
}
}
async fn plain_events_inner(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!("httpret plain_events_inner req: {:?}", req);
info!("httpret plain_events req: {:?}", req);
let accept_def = APP_JSON;
let accept = req
.headers()
@@ -85,6 +77,7 @@ async fn plain_events_json(req: Request<Body>, node_config: &NodeConfigCached) -
query.disk_io_buffer_size(),
query.timeout(),
node_config.clone(),
query.events_max().unwrap_or(u64::MAX),
query.do_log(),
);
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), AggKind::Plain, node_config).await?;

13
httpret/src/evinfo.rs
View File

@@ -74,7 +74,12 @@ impl EventInfoScan {
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>, Error> {
let ret = channel_exec(
EvInfoFunc::new(query.clone(), query.timeout(), node_config.clone()),
EvInfoFunc::new(
query.clone(),
query.timeout(),
query.events_max().unwrap_or(u64::MAX),
node_config.clone(),
),
query.channel(),
query.range(),
AggKind::Stats1,
@@ -89,13 +94,15 @@ pub struct EvInfoFunc {
query: PlainEventsJsonQuery,
timeout: Duration,
node_config: NodeConfigCached,
events_max: u64,
}
impl EvInfoFunc {
pub fn new(query: PlainEventsJsonQuery, timeout: Duration, node_config: NodeConfigCached) -> Self {
pub fn new(query: PlainEventsJsonQuery, timeout: Duration, events_max: u64, node_config: NodeConfigCached) -> Self {
Self {
query,
timeout,
events_max,
node_config,
}
}
@@ -151,7 +158,7 @@ impl ChannelExecFunction for EvInfoFunc {
// TODO Must issue multiple reads to GPFS, keep futures in a ordered queue.
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster);
let f = collect_plain_events_json(s, self.timeout, 0, self.query.do_log());
let f = collect_plain_events_json(s, self.timeout, 0, self.events_max, self.query.do_log());
let f = FutureExt::map(f, |item| match item {
Ok(item) => {
// TODO add channel entry info here?

8
httpret/src/httpret.rs
View File

@@ -225,7 +225,7 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/4/random_channel" {
} else if path == "/api/4/random/channel" {
if req.method() == Method::GET {
Ok(random_channel(req, &node_config).await?)
} else {
@@ -287,7 +287,10 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
}
} else if path == "/api/4/channel/config" {
if req.method() == Method::GET {
Ok(channel_config(req, &node_config).await?)
match channel_config(req, &node_config).await {
Ok(k) => Ok(k),
Err(e) => Ok(e.to_public_response()),
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
@@ -451,6 +454,7 @@ trait ToPublicResponse {
impl ToPublicResponse for Error {
fn to_public_response(&self) -> Response<Body> {
error!("ToPublicResponse converts: {self:?}");
use std::fmt::Write;
let status = match self.reason() {
Some(::err::Reason::BadRequest) => StatusCode::BAD_REQUEST,

11
httpret/src/proxy.rs
View File

@@ -4,7 +4,7 @@ use crate::api1::{channel_search_configs_v1, channel_search_list_v1, gather_json
use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::pulsemap::MapPulseQuery;
use crate::{api_1_docs, api_4_docs, response, Cont};
use crate::{api_1_docs, api_4_docs, response, response_err, Cont};
use disk::events::PlainEventsJsonQuery;
use futures_core::Stream;
use futures_util::pin_mut;
@@ -471,7 +471,14 @@ where
Some(v) => {
if v == APP_JSON || v == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", head.uri))?;
let query = QT::from_url(&url)?;
let query = match QT::from_url(&url) {
Ok(k) => k,
Err(_) => {
let msg = format!("Malformed request or missing parameters");
return Ok(response_err(StatusCode::BAD_REQUEST, msg)?);
}
};
// TODO is this special case used any more?
let sh = if url.as_str().contains("/map/pulse/") {
get_query_host_for_backend_2(&query.backend(), proxy_config)?
} else {