daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Forward non-200 status in proxy. Start with event stats reader

Browse Source
This commit is contained in:
Dominik Werder
2022-02-07 21:35:25 +01:00
parent bcd3273dea
commit a9f9d1ada6
35 changed files with 913 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
httpret/src/api1.rs
View File

@@ -122,14 +122,19 @@ pub async fn channel_search_list_v1(req: Request<Body>, proxy_config: &ProxyConf
a
})?;
let tags: Vec<_> = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let nt = |tag, res| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
let res: ChannelSearchResult = match serde_json::from_slice(&body) {
Ok(k) => k,
Err(_) => ChannelSearchResult { channels: vec![] },
};
Ok(res)
let ret = SubRes {
tag,
status: StatusCode::OK,
val: res,
};
Ok(ret)
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
};
@@ -216,14 +221,19 @@ pub async fn channel_search_configs_v1(
a
})?;
let tags: Vec<_> = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let nt = |tag, res| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
let res: ChannelSearchResult = match serde_json::from_slice(&body) {
Ok(k) => k,
Err(_) => ChannelSearchResult { channels: vec![] },
};
Ok(res)
let ret = SubRes {
tag,
status: StatusCode::OK,
val: res,
};
Ok(ret)
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
};

171
httpret/src/evinfo.rs Normal file
View File

@@ -0,0 +1,171 @@
use crate::err::Error;
use crate::response;
use bytes::Bytes;
use disk::channelexec::channel_exec;
use disk::channelexec::collect_plain_events_json;
use disk::channelexec::ChannelExecFunction;
use disk::decode::Endianness;
use disk::decode::EventValueFromBytes;
use disk::decode::EventValueShape;
use disk::decode::NumFromBytes;
use disk::events::PlainEventsJsonQuery;
use disk::merge::mergedfromremotes::MergedFromRemotes;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::TryStreamExt;
use http::{Method, StatusCode};
use hyper::{Body, Request, Response};
use items::numops::NumOps;
use items::streams::Collectable;
use items::Clearable;
use items::EventsNodeProcessor;
use items::Framable;
use items::FrameType;
use items::PushableIndex;
use items::Sitemty;
use items::TimeBinnableType;
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::AggKind;
use netpod::Channel;
use netpod::NanoRange;
use netpod::NodeConfigCached;
use netpod::PerfOpts;
use netpod::ScalarType;
use netpod::Shape;
use serde::de::DeserializeOwned;
use std::fmt::Debug;
use std::pin::Pin;
use std::time::Duration;
use url::Url;
pub struct EventInfoScan {}
impl EventInfoScan {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path().starts_with("/api/4/event/info") {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!("EventInfoScan::handle");
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
let (head, _body) = req.into_parts();
let url = Url::parse(&format!("dummy:{}", head.uri))?;
let query = PlainEventsJsonQuery::from_url(&url)?;
let ret = match Self::exec(&query, node_config).await {
Ok(stream) => {
//
let stream = stream.map_ok(|_| Bytes::new());
response(StatusCode::OK).body(Body::wrap_stream(stream))?
}
Err(e) => response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::from(format!("{:?}", e)))?,
};
Ok(ret)
}
pub async fn exec(
query: &PlainEventsJsonQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>, Error> {
let ret = channel_exec(
EvInfoFunc::new(query.clone(), query.timeout(), node_config.clone()),
query.channel(),
query.range(),
AggKind::Stats1,
node_config,
)
.await?;
Ok(Box::pin(ret.map_err(Error::from)))
}
}
pub struct EvInfoFunc {
query: PlainEventsJsonQuery,
timeout: Duration,
node_config: NodeConfigCached,
}
impl EvInfoFunc {
pub fn new(query: PlainEventsJsonQuery, timeout: Duration, node_config: NodeConfigCached) -> Self {
Self {
query,
timeout,
node_config,
}
}
pub fn channel(&self) -> &Channel {
&self.query.channel()
}
pub fn range(&self) -> &NanoRange {
&self.query.range()
}
}
impl ChannelExecFunction for EvInfoFunc {
type Output = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
fn exec<NTY, END, EVS, ENP>(
self,
byte_order: END,
_scalar_type: ScalarType,
_shape: Shape,
event_value_shape: EVS,
_events_node_proc: ENP,
) -> Result<Self::Output, ::err::Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Batch> + 'static,
// TODO require these things in general?
<ENP as EventsNodeProcessor>::Output: Debug + Collectable + PushableIndex + Clearable,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output: Debug
+ TimeBinnableType<Output = <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>
+ Collectable
+ Unpin,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
FrameType + Framable + DeserializeOwned,
{
let _ = byte_order;
let _ = event_value_shape;
let perf_opts = PerfOpts { inmem_bufcap: 4096 };
let evq = RawEventsQuery {
channel: self.query.channel().clone(),
range: self.query.range().clone(),
agg_kind: AggKind::Plain,
disk_io_buffer_size: self.query.disk_io_buffer_size(),
do_decompress: true,
};
// TODO Use a Merged-From-Multiple-Local-Splits.
// TODO Pass the read buffer size from query parameter: GPFS needs a larger buffer..
// TODO Must issue multiple reads to GPFS, keep futures in a ordered queue.
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster);
let f = collect_plain_events_json(s, self.timeout, 0, self.query.do_log());
let f = FutureExt::map(f, |item| match item {
Ok(item) => {
// TODO add channel entry info here?
//let obj = item.as_object_mut().unwrap();
//obj.insert("channelName", JsonValue::String(en));
Ok(Bytes::from(serde_json::to_vec(&item)?))
}
Err(e) => Err(e.into()),
});
let s = futures_util::stream::once(f);
Ok(Box::pin(s))
}
fn empty() -> Self::Output {
Box::pin(futures_util::stream::empty())
}
}

39
httpret/src/gather.rs
View File

@@ -4,6 +4,7 @@ use futures_util::{select, FutureExt};
use http::{Method, StatusCode};
use hyper::{Body, Client, Request, Response};
use hyper_tls::HttpsConnector;
use netpod::log::*;
use netpod::{Node, NodeConfigCached, APP_JSON};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
@@ -164,6 +165,7 @@ pub async fn gather_get_json(req: Request<Body>, node_config: &NodeConfigCached)
pub struct SubRes<T> {
pub tag: String,
pub status: StatusCode,
pub val: T,
}
@@ -178,7 +180,11 @@ pub async fn gather_get_json_generic<SM, NT, FT>(
) -> Result<Response<Body>, Error>
where
SM: Send + 'static,
NT: Fn(Response<Body>) -> Pin<Box<dyn Future<Output = Result<SM, Error>> + Send>> + Send + Sync + Copy + 'static,
NT: Fn(String, Response<Body>) -> Pin<Box<dyn Future<Output = Result<SubRes<SM>, Error>> + Send>>
+ Send
+ Sync
+ Copy
+ 'static,
FT: Fn(Vec<SubRes<SM>>) -> Result<Response<Body>, Error>,
{
assert!(urls.len() == bodies.len());
@@ -223,10 +229,7 @@ where
client.request(req?).fuse()
}
} => {
let ret = SubRes {
tag: tag,
val:nt(res?).await?,
};
let ret = nt(tag, res?).await?;
Ok(ret)
}
}
@@ -234,18 +237,23 @@ where
(url, task)
})
.collect();
let mut a = vec![];
let mut a: Vec<SubRes<SM>> = vec![];
for (_schemehostport, jh) in spawned {
let res = match jh.await {
let res: SubRes<SM> = match jh.await {
Ok(k) => match k {
Ok(k) => k,
Err(e) => return Err(e),
Err(e) => {
warn!("{e:?}");
return Err(e);
}
},
Err(e) => return Err(e.into()),
Err(e) => {
warn!("{e:?}");
return Err(e.into());
}
};
a.push(res);
}
let a = a;
ft(a)
}
@@ -260,8 +268,15 @@ mod test {
vec![],
vec![],
vec![],
|_res| {
let fut = async { Ok(()) };
|tag, _res| {
let fut = async {
let ret = SubRes {
tag,
status: StatusCode::OK,
val: (),
};
Ok(ret)
};
Box::pin(fut)
},
|_all| {

15
httpret/src/httpret.rs
View File

@@ -1,6 +1,7 @@
pub mod api1;
pub mod channelarchiver;
pub mod err;
pub mod evinfo;
pub mod gather;
pub mod proxy;
pub mod pulsemap;
@@ -50,14 +51,20 @@ pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
let addr = SocketAddr::from_str(&format!("{}:{}", node_config.node.listen, node_config.node.port))?;
let make_service = make_service_fn({
move |conn: &AddrStream| {
// TODO send to logstash
debug!("new connection from {:?}", conn.remote_addr());
let node_config = node_config.clone();
let addr = conn.remote_addr();
async move {
Ok::<_, Error>(service_fn({
move |req| {
info!("REQUEST {:?} {:?}", addr, req.uri());
// TODO send to logstash
info!(
"REQUEST {:?} - {:?} - {:?} - {:?}",
addr,
req.method(),
req.uri(),
req.headers()
);
let f = http_service(req, node_config.clone());
Cont { f: Box::pin(f) }
}
@@ -170,8 +177,6 @@ macro_rules! static_http_api1 {
}
async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
// TODO send to logstash
debug!("http_service_try {:?}", req.uri());
let uri = req.uri().clone();
let path = uri.path();
if path == "/api/4/node_status" {
@@ -289,6 +294,8 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if let Some(h) = evinfo::EventInfoScan::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = pulsemap::IndexFullHttpFunction::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = pulsemap::MarkClosedHttpFunction::handler(&req) {

175
httpret/src/proxy.rs
View File

@@ -41,6 +41,14 @@ pub async fn proxy(proxy_config: ProxyConfig) -> Result<(), Error> {
async move {
Ok::<_, Error>(service_fn({
move |req| {
// TODO send to logstash
info!(
"REQUEST {:?} - {:?} - {:?} - {:?}",
addr,
req.method(),
req.uri(),
req.headers()
);
let f = proxy_http_service(req, proxy_config.clone());
Cont { f: Box::pin(f) }
}
@@ -119,6 +127,26 @@ async fn proxy_http_service_try(req: Request<Body>, proxy_config: &ProxyConfig)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path.starts_with("/api/4/test/http/204") {
Ok(response(StatusCode::NO_CONTENT).body(Body::from("No Content"))?)
} else if path.starts_with("/api/4/test/http/400") {
Ok(response(StatusCode::BAD_REQUEST).body(Body::from("Bad Request"))?)
} else if path.starts_with("/api/4/test/http/405") {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::from("Method Not Allowed"))?)
} else if path.starts_with("/api/4/test/http/406") {
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::from("Not Acceptable"))?)
} else if path.starts_with("/api/4/test/log/error") {
error!("{path}");
Ok(response(StatusCode::OK).body(Body::empty())?)
} else if path.starts_with("/api/4/test/log/warn") {
warn!("{path}");
Ok(response(StatusCode::OK).body(Body::empty())?)
} else if path.starts_with("/api/4/test/log/info") {
info!("{path}");
Ok(response(StatusCode::OK).body(Body::empty())?)
} else if path.starts_with("/api/4/test/log/debug") {
debug!("{path}");
Ok(response(StatusCode::OK).body(Body::empty())?)
} else if path.starts_with(DISTRI_PRE) {
proxy_distribute_v2(req).await
} else {
@@ -241,55 +269,74 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
});
}
let tags = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let nt = |tag, res| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
//info!("got a result {:?}", body);
let res: ChannelSearchResult = match serde_json::from_slice::<ChannelSearchResult>(&body) {
Ok(k) => k,
Err(_) => {
#[derive(Deserialize)]
struct ResItemApi0 {
name: String,
source: String,
backend: String,
#[serde(rename = "type")]
ty: String,
let res: SubRes<ChannelSearchResult> =
match serde_json::from_slice::<ChannelSearchResult>(&body) {
Ok(val) => {
let ret = SubRes {
tag,
status: StatusCode::OK,
val,
};
ret
}
#[derive(Deserialize)]
struct ResContApi0 {
#[allow(dead_code)]
backend: String,
channels: Vec<ResItemApi0>,
}
match serde_json::from_slice::<Vec<ResContApi0>>(&body) {
Ok(k) => {
let mut a = vec![];
if let Some(g) = k.first() {
for c in &g.channels {
let z = ChannelSearchSingleResult {
backend: c.backend.clone(),
description: String::new(),
name: c.name.clone(),
shape: vec![],
source: c.source.clone(),
ty: c.ty.clone(),
unit: String::new(),
is_api_0: Some(true),
};
a.push(z);
Err(_) => {
#[derive(Deserialize)]
struct ResItemApi0 {
name: String,
source: String,
backend: String,
#[serde(rename = "type")]
ty: String,
}
#[derive(Deserialize)]
struct ResContApi0 {
#[allow(dead_code)]
backend: String,
channels: Vec<ResItemApi0>,
}
match serde_json::from_slice::<Vec<ResContApi0>>(&body) {
Ok(k) => {
let mut a = vec![];
if let Some(g) = k.first() {
for c in &g.channels {
let z = ChannelSearchSingleResult {
backend: c.backend.clone(),
description: String::new(),
name: c.name.clone(),
shape: vec![],
source: c.source.clone(),
ty: c.ty.clone(),
unit: String::new(),
is_api_0: Some(true),
};
a.push(z);
}
}
let ret = ChannelSearchResult { channels: a };
let ret = SubRes {
tag,
status: StatusCode::OK,
val: ret,
};
ret
}
Err(_) => {
error!("Channel search response parse failed");
let ret = ChannelSearchResult { channels: vec![] };
let ret = SubRes {
tag,
status: StatusCode::OK,
val: ret,
};
ret
}
let ret = ChannelSearchResult { channels: a };
ret
}
Err(_) => {
error!("Channel search response parse failed");
ChannelSearchResult { channels: vec![] }
}
}
}
};
};
Ok(res)
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
@@ -428,22 +475,46 @@ where
a
})?;
let tags: Vec<_> = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let nt = |tag: String, res: Response<Body>| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
match serde_json::from_slice::<JsonValue>(&body) {
Ok(k) => Ok(k),
Err(e) => Err(e.into()),
let (head, body) = res.into_parts();
if head.status == StatusCode::OK {
let body = hyper::body::to_bytes(body).await?;
match serde_json::from_slice::<JsonValue>(&body) {
Ok(val) => {
let ret = SubRes {
tag,
status: head.status,
val,
};
Ok(ret)
}
Err(e) => {
warn!("can not parse response: {e:?}");
Err(e.into())
}
}
} else {
let body = hyper::body::to_bytes(body).await?;
let b = String::from_utf8_lossy(&body);
let ret = SubRes {
tag,
status: head.status,
// TODO would like to pass arbitrary type of body in these cases:
val: serde_json::Value::String(format!("{}", b)),
};
Ok(ret)
}
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
Box::pin(fut) as Pin<Box<dyn Future<Output = Result<SubRes<serde_json::Value>, Error>> + Send>>
};
let ft = |mut all: Vec<SubRes<JsonValue>>| {
if all.len() > 0 {
all.truncate(1);
let z = all.pop().unwrap();
let res = z.val;
let res = response(StatusCode::OK)
// TODO want to pass arbitrary body type:
let res = response(z.status)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&res)?))?;
return Ok(res);
@@ -472,9 +543,11 @@ fn get_query_host_for_backend(backend: &str, proxy_config: &ProxyConfig) -> Resu
return Err(Error::with_msg(format!("host not found for backend {:?}", backend)));
}
fn get_query_host_for_backend_2(backend: &str, _proxy_config: &ProxyConfig) -> Result<String, Error> {
if backend == "sf-databuffer" {
return Ok("https://sf-data-api.psi.ch".into());
fn get_query_host_for_backend_2(backend: &str, proxy_config: &ProxyConfig) -> Result<String, Error> {
for back in &proxy_config.backends2 {
if back.name == backend {
return Ok(back.url.clone());
}
}
return Err(Error::with_msg(format!("host not found for backend {:?}", backend)));
}

9
httpret/src/proxy/api4.rs
View File

@@ -42,7 +42,7 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
a
})?;
let tags = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let nt = |tag, res| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
//info!("got a result {:?}", body);
@@ -54,7 +54,12 @@ pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> R
return Err(Error::with_msg_no_trace(msg));
}
};
Ok(res)
let ret = SubRes {
tag,
status: StatusCode::OK,
val: res,
};
Ok(ret)
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
};

5
httpret/src/pulsemap.rs
View File

@@ -326,7 +326,7 @@ async fn update_task(do_abort: Arc<AtomicUsize>, node_config: NodeConfigCached)
info!("update_task break A");
break;
}
tokio::time::sleep(Duration::from_millis(10000)).await;
tokio::time::sleep(Duration::from_millis(60000)).await;
if do_abort.load(Ordering::SeqCst) != 0 {
info!("update_task break B");
break;
@@ -464,7 +464,6 @@ impl FromUrl for MapPulseQuery {
.into();
let pulse: u64 = pulsestr.parse()?;
let ret = Self { backend, pulse };
info!("GOT {:?}", ret);
Ok(ret)
}
}
@@ -647,7 +646,7 @@ impl Api4MapPulseHttpFunction {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("Api4MapPulseHttpFunction handle uri: {:?}", req.uri());
//info!("Api4MapPulseHttpFunction handle uri: {:?}", req.uri());
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = MapPulseQuery::from_url(&url)?;
let histo = MapPulseHistoHttpFunction::histo(q.pulse, node_config).await?;