daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP Read3

Browse Source
This commit is contained in:
Dominik Werder
2022-03-02 21:40:03 +01:00
parent 0b741d187e
commit d67608fabc
9 changed files with 426 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
httpret/src/api1.rs
View File

@@ -464,31 +464,6 @@ async fn process_answer(res: Response<Body>) -> Result<JsonValue, Error> {
}
}
pub async fn proxy_distribute_v1(req: Request<Body>) -> Result<Response<Body>, Error> {
let (mut sink, body) = Body::channel();
let uri = format!("http://sf-daqbuf-33:8371{}", req.uri().path());
let res = Response::builder().status(StatusCode::OK).body(body)?;
tokio::spawn(async move {
let req = Request::builder().method(Method::GET).uri(uri).body(Body::empty())?;
let res = Client::new().request(req).await?;
if res.status() == StatusCode::OK {
let (_heads, mut body) = res.into_parts();
loop {
use hyper::body::HttpBody;
let chunk = body.data().await;
if let Some(k) = chunk {
let k = k?;
sink.send_data(k).await?;
} else {
break;
}
}
}
Ok::<_, Error>(())
});
Ok(res)
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Api1Range {
#[serde(rename = "startDate")]
@@ -696,12 +671,15 @@ impl Stream for DataApiPython3DataStream {
self.data_done = true;
let mut sb = crate::status_board().unwrap();
sb.add_error(&self.status_id, e);
// TODO format as python data api error frame:
//let mut buf = BytesMut::with_capacity(1024);
//buf.put_slice("".as_bytes());
//Ready(Some(Ok(buf)))
self.data_done = true;
Ready(None)
if false {
// TODO format as python data api error frame:
let mut buf = BytesMut::with_capacity(1024);
buf.put_slice("".as_bytes());
Ready(Some(Ok(buf)))
} else {
self.data_done = true;
Ready(None)
}
}
},
None => {
@@ -740,7 +718,7 @@ impl Stream for DataApiPython3DataStream {
let perf_opts = PerfOpts { inmem_bufcap: 1024 * 4 };
// TODO is this a good to place decide this?
let s = if self.node_config.node_config.cluster.is_central_storage {
debug!("Set up central storage stream");
info!("Set up central storage stream");
// TODO pull up this config
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
let s = disk::raw::conn::make_local_event_blobs_stream(
@@ -755,6 +733,11 @@ impl Stream for DataApiPython3DataStream {
)?;
Box::pin(s) as Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>
} else {
if let Some(sh) = &entry.shape {
if sh.len() > 1 {
warn!("Remote stream fetch for shape {sh:?}");
}
}
debug!("Set up merged remote stream");
let s = disk::merge::mergedblobsfromremotes::MergedBlobsFromRemotes::new(
evq,
@@ -859,13 +842,13 @@ impl Api1EventsBinaryHandler {
if req.method() != Method::POST {
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?);
}
let accept = req
.headers()
let (head, body) = req.into_parts();
let accept = head
.headers
.get(http::header::ACCEPT)
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
.to_owned();
let (_head, body) = req.into_parts();
let body_data = hyper::body::to_bytes(body).await?;
let qu: Api1Query = if let Ok(qu) = serde_json::from_slice(&body_data) {
qu
@@ -926,3 +909,44 @@ impl Api1EventsBinaryHandler {
Ok(ret)
}
}
pub struct RequestStatusHandler {}
impl RequestStatusHandler {
pub fn path_prefix() -> &'static str {
"/api/1/requestStatus/"
}
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path().starts_with(Self::path_prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, body) = req.into_parts();
if head.method != Method::GET {
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?);
}
let accept = head
.headers
.get(http::header::ACCEPT)
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
.to_owned();
if accept != APP_JSON && accept != ACCEPT_ALL {
// TODO set the public error code and message and return Err(e).
let e = Error::with_public_msg(format!("Unsupported Accept: {:?}", accept));
error!("{e:?}");
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
let _body_data = hyper::body::to_bytes(body).await?;
let status_id = &head.uri.path()[Self::path_prefix().len()..];
info!("RequestStatusHandler status_id {:?}", status_id);
let s = crate::status_board()?.status_as_json(status_id);
let ret = response(StatusCode::OK).body(Body::from(s))?;
Ok(ret)
}
}

42
httpret/src/httpret.rs
View File

@@ -211,6 +211,8 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if let Some(h) = StatusBoardAllHandler::handler(&req) {
h.handle(req, &node_config).await
} else if path == "/api/4/search/channel" {
if req.method() == Method::GET {
Ok(search::channel_search(req, &node_config).await?)
@@ -331,8 +333,8 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockStream::handler(path) {
h.handle(req, &node_config).await
} else if path.starts_with("/api/1/requestStatus/") {
Ok(response(StatusCode::OK).body(Body::from("{}"))?)
} else if let Some(h) = api1::RequestStatusHandler::handler(&req) {
h.handle(req, &node_config).await
} else if path.starts_with("/api/1/documentation/") {
if req.method() == Method::GET {
api_1_docs(path)
@@ -766,8 +768,11 @@ pub struct StatusBoardEntry {
ts_created: SystemTime,
#[serde(serialize_with = "instant_serde::ser")]
ts_updated: SystemTime,
#[serde(skip_serializing_if = "items::bool_is_false")]
is_error: bool,
#[serde(skip_serializing_if = "items::bool_is_false")]
is_ok: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
errors: Vec<Error>,
}
@@ -777,7 +782,7 @@ mod instant_serde {
pub fn ser<S: Serializer>(x: &SystemTime, ser: S) -> Result<S::Ok, S::Error> {
let dur = x.duration_since(std::time::UNIX_EPOCH).unwrap();
let dt = chrono::TimeZone::timestamp(&chrono::Utc, dur.as_secs() as i64, dur.subsec_nanos());
let s = dt.format("%Y-%m-%d %H:%M:%S").to_string();
let s = dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string();
ser.serialize_str(&s)
}
}
@@ -815,6 +820,7 @@ impl StatusBoard {
f.read_exact(&mut buf).unwrap();
let n = u64::from_le_bytes(buf);
let s = format!("{:016x}", n);
info!("new_status_id {s}");
self.entries.insert(s.clone(), StatusBoardEntry::new());
s
}
@@ -868,6 +874,36 @@ impl StatusBoard {
}
}
}
pub fn status_as_json(&self, status_id: &str) -> String {
#[derive(Serialize)]
struct StatJs {
#[serde(skip_serializing_if = "Vec::is_empty")]
errors: Vec<::err::PublicError>,
}
match self.entries.get(status_id) {
Some(e) => {
if e.is_ok {
let js = StatJs { errors: vec![] };
return serde_json::to_string(&js).unwrap();
} else if e.is_error {
let errors = e.errors.iter().map(|e| (&e.0).into()).collect();
let js = StatJs { errors };
return serde_json::to_string(&js).unwrap();
} else {
warn!("requestStatus for unfinished {status_id}");
let js = StatJs { errors: vec![] };
return serde_json::to_string(&js).unwrap();
}
}
None => {
error!("can not find status id {}", status_id);
let e = ::err::Error::with_public_msg_no_trace(format!("Request status ID unknown {status_id}"));
let js = StatJs { errors: vec![e.into()] };
return serde_json::to_string(&js).unwrap();
}
}
}
}
static STATUS_BOARD: AtomicPtr<RwLock<StatusBoard>> = AtomicPtr::new(std::ptr::null_mut());

8
httpret/src/proxy.rs
View File

@@ -1,6 +1,6 @@
pub mod api4;
use crate::api1::{channel_search_configs_v1, channel_search_list_v1, gather_json_2_v1, proxy_distribute_v1};
use crate::api1::{channel_search_configs_v1, channel_search_list_v1, gather_json_2_v1};
use crate::err::Error;
use crate::gather::{gather_get_json_generic, SubRes};
use crate::pulsemap::MapPulseQuery;
@@ -13,11 +13,11 @@ use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Request, Response, Server};
use hyper_tls::HttpsConnector;
use itertools::Itertools;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::{log::*, ACCEPT_ALL};
use netpod::{
AppendToUrl, ChannelConfigQuery, ChannelSearchQuery, ChannelSearchResult, ChannelSearchSingleResult, FromUrl,
HasBackend, HasTimeout, ProxyConfig, APP_JSON,
HasBackend, HasTimeout, ProxyConfig, ACCEPT_ALL, APP_JSON,
};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
@@ -115,8 +115,6 @@ async fn proxy_http_service_try(req: Request<Body>, proxy_config: &ProxyConfig)
Ok(proxy_single_backend_query::<BinnedQuery>(req, proxy_config).await?)
} else if path == "/api/4/channel/config" {
Ok(proxy_single_backend_query::<ChannelConfigQuery>(req, proxy_config).await?)
} else if path.starts_with("/distribute") {
proxy_distribute_v1(req).await
} else if path.starts_with("/api/1/documentation/") {
if req.method() == Method::GET {
api_1_docs(path)