daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on adding stats to the streams

Browse Source
This commit is contained in:
Dominik Werder
2021-05-01 18:00:05 +02:00
parent 0f9408e9f8
commit 0eb6b3364d
15 changed files with 275 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
httpret/src/lib.rs
View File

@@ -9,7 +9,7 @@ use http::{HeaderMap, Method, StatusCode};
use hyper::service::{make_service_fn, service_fn};
use hyper::{server::Server, Body, Request, Response};
use net::SocketAddr;
use netpod::{Node, NodeConfig};
use netpod::{Node, NodeConfigCached};
use panic::{AssertUnwindSafe, UnwindSafe};
use pin::Pin;
use std::{future, net, panic, pin, task};
@@ -18,21 +18,19 @@ use tracing::field::Empty;
#[allow(unused_imports)]
use tracing::{debug, error, info, span, trace, warn, Level};
pub async fn host(node_config: NodeConfig, node: Node) -> Result<(), Error> {
pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
let node_config = node_config.clone();
let node = node.clone();
let rawjh = taskrun::spawn(raw_service(node_config.clone(), node.clone()));
let rawjh = taskrun::spawn(raw_service(node_config.clone()));
use std::str::FromStr;
let addr = SocketAddr::from_str(&format!("{}:{}", node.listen, node.port))?;
let addr = SocketAddr::from_str(&format!("{}:{}", node_config.node.listen, node_config.node.port))?;
let make_service = make_service_fn({
move |conn| {
info!("new raw {:?}", conn);
let node_config = node_config.clone();
let node = node.clone();
async move {
Ok::<_, Error>(service_fn({
move |req| {
let f = data_api_proxy(req, node_config.clone(), node.clone());
let f = data_api_proxy(req, node_config.clone());
Cont { f: Box::pin(f) }
}
}))
@@ -44,8 +42,8 @@ pub async fn host(node_config: NodeConfig, node: Node) -> Result<(), Error> {
Ok(())
}
async fn data_api_proxy(req: Request<Body>, node_config: NodeConfig, node: Node) -> Result<Response<Body>, Error> {
match data_api_proxy_try(req, &node_config, &node).await {
async fn data_api_proxy(req: Request<Body>, node_config: NodeConfigCached) -> Result<Response<Body>, Error> {
match data_api_proxy_try(req, &node_config).await {
Ok(k) => Ok(k),
Err(e) => {
error!("{:?}", e);
@@ -84,28 +82,24 @@ where
impl<F> UnwindSafe for Cont<F> {}
async fn data_api_proxy_try(
req: Request<Body>,
node_config: &NodeConfig,
node: &Node,
) -> Result<Response<Body>, Error> {
async fn data_api_proxy_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let uri = req.uri().clone();
let path = uri.path();
if path == "/api/1/parsed_raw" {
if req.method() == Method::POST {
Ok(parsed_raw(req, node).await?)
Ok(parsed_raw(req, &node_config.node).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/1/binned" {
if req.method() == Method::GET {
Ok(binned(req, &node_config, node).await?)
Ok(binned(req, node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/1/prebinned" {
if req.method() == Method::GET {
Ok(prebinned(req, &node_config, node).await?)
Ok(prebinned(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
@@ -224,10 +218,10 @@ where
}
}
async fn binned(req: Request<Body>, node_config: &NodeConfig, node: &Node) -> Result<Response<Body>, Error> {
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
let query = disk::cache::Query::from_request(&head)?;
let ret = match disk::cache::binned_bytes_for_http(node_config, node, &query).await {
let ret = match disk::cache::binned_bytes_for_http(node_config, &query).await {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(s, format!("desc-BINNED")))?,
Err(e) => {
error!("{:?}", e);
@@ -237,14 +231,14 @@ async fn binned(req: Request<Body>, node_config: &NodeConfig, node: &Node) -> Re
Ok(ret)
}
async fn prebinned(req: Request<Body>, node_config: &NodeConfig, node: &Node) -> Result<Response<Body>, Error> {
async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
let q = PreBinnedQuery::from_request(&head)?;
let desc = format!("pre-b-{}", q.patch.bin_t_len() / 1000000000);
let span1 = span!(Level::INFO, "httpret::prebinned", desc = &desc.as_str());
span1.in_scope(|| {
trace!("prebinned");
let ret = match disk::cache::pre_binned_bytes_for_http(node_config, node, &q) {
let ret = match disk::cache::pre_binned_bytes_for_http(node_config, &q) {
Ok(s) => response(StatusCode::OK).body(BodyStream::wrapped(
s,
format!(