daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Imagebuffer reads, time binning

Browse Source
This commit is contained in:
Dominik Werder
2023-01-19 20:05:25 +01:00
parent 9c68476626
commit 8495853f8e
26 changed files with 341 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
httpret/src/api1.rs
View File

@@ -611,7 +611,6 @@ pub struct DataApiPython3DataStream {
chan_stream: Option<Pin<Box<dyn Stream<Item = Result<BytesMut, Error>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<Config, Error>> + Send>>>,
disk_io_tune: DiskIoTune,
#[allow(unused)]
do_decompress: bool,
#[allow(unused)]
event_count: u64,
@@ -696,12 +695,13 @@ impl DataApiPython3DataStream {
compression,
};
let h = serde_json::to_string(&head)?;
debug!("sending channel header {}", h);
info!("sending channel header {}", h);
let l1 = 1 + h.as_bytes().len() as u32;
d.put_u32(l1);
d.put_u8(0);
debug!("header frame byte len {}", 4 + 1 + h.as_bytes().len());
info!("header frame byte len {}", 4 + 1 + h.as_bytes().len());
d.extend_from_slice(h.as_bytes());
d.put_u32(l1);
*header_out = true;
}
match &b.shapes[i1] {
@@ -712,6 +712,7 @@ impl DataApiPython3DataStream {
d.put_u64(b.tss[i1]);
d.put_u64(b.pulses[i1]);
d.put_slice(&b.blobs[i1]);
d.put_u32(l1);
}
}
*count_events += 1;
@@ -806,7 +807,7 @@ impl Stream for DataApiPython3DataStream {
evq.channel().clone(),
&entry,
evq.agg_kind().need_expand(),
true,
self.do_decompress,
event_chunker_conf,
self.disk_io_tune.clone(),
&self.node_config,
@@ -937,8 +938,16 @@ impl Api1EventsBinaryHandler {
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
.to_owned();
let body_data = hyper::body::to_bytes(body).await?;
let qu: Api1Query = match serde_json::from_slice(&body_data) {
Ok(qu) => qu,
if body_data.len() < 512 && body_data.first() == Some(&"{".as_bytes()[0]) {
info!("request body_data string: {}", String::from_utf8_lossy(&body_data));
}
let qu = match serde_json::from_slice::<Api1Query>(&body_data) {
Ok(mut qu) => {
if node_config.node_config.cluster.is_central_storage {
qu.set_decompress(false);
}
qu
}
Err(e) => {
error!("got body_data: {:?}", String::from_utf8_lossy(&body_data[..]));
error!("can not parse: {e}");
@@ -976,42 +985,43 @@ impl Api1EventsBinaryHandler {
trace!("Api1Query beg_date {:?} end_date {:?}", beg_date, end_date);
//let url = Url::parse(&format!("dummy:{}", req.uri()))?;
//let query = PlainEventsBinaryQuery::from_url(&url)?;
if accept != APP_OCTET && accept != ACCEPT_ALL {
if accept.contains(APP_OCTET) || accept.contains(ACCEPT_ALL) {
let beg = beg_date.timestamp() as u64 * SEC + beg_date.timestamp_subsec_nanos() as u64;
let end = end_date.timestamp() as u64 * SEC + end_date.timestamp_subsec_nanos() as u64;
let range = NanoRange { beg, end };
// TODO check for valid given backend name:
let backend = &node_config.node_config.cluster.backend;
let chans = qu
.channels()
.iter()
.map(|ch| Channel {
backend: backend.into(),
name: ch.name().into(),
series: None,
})
.collect();
// TODO use a better stream protocol with built-in error delivery.
let status_id = super::status_board()?.new_status_id();
let s = DataApiPython3DataStream::new(
range.clone(),
chans,
qu.disk_io_tune().clone(),
qu.decompress(),
qu.events_max().unwrap_or(u64::MAX),
status_id.clone(),
node_config.clone(),
);
let s = s.instrument(span);
let body = BodyStream::wrapped(s, format!("Api1EventsBinaryHandler"));
let ret = response(StatusCode::OK).header("x-daqbuffer-request-id", status_id);
let ret = ret.body(body)?;
Ok(ret)
} else {
// TODO set the public error code and message and return Err(e).
let e = Error::with_public_msg(format!("Unsupported Accept: {:?}", accept));
error!("{e:?}");
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
let beg = beg_date.timestamp() as u64 * SEC + beg_date.timestamp_subsec_nanos() as u64;
let end = end_date.timestamp() as u64 * SEC + end_date.timestamp_subsec_nanos() as u64;
let range = NanoRange { beg, end };
// TODO check for valid given backend name:
let backend = &node_config.node_config.cluster.backend;
let chans = qu
.channels()
.iter()
.map(|ch| Channel {
backend: backend.into(),
name: ch.name().into(),
series: None,
})
.collect();
// TODO use a better stream protocol with built-in error delivery.
let status_id = super::status_board()?.new_status_id();
let s = DataApiPython3DataStream::new(
range.clone(),
chans,
qu.disk_io_tune().clone(),
qu.decompress(),
qu.events_max().unwrap_or(u64::MAX),
status_id.clone(),
node_config.clone(),
);
let s = s.instrument(span);
let body = BodyStream::wrapped(s, format!("Api1EventsBinaryHandler"));
let ret = response(StatusCode::OK).header("x-daqbuffer-request-id", status_id);
let ret = ret.body(body)?;
Ok(ret)
}
}

13
httpret/src/api4/binned.rs
View File

@@ -18,6 +18,7 @@ async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCache
debug!("httpret plain_events_json req: {:?}", req);
let (_head, _body) = req.into_parts();
let query = BinnedQuery::from_url(&url).map_err(|e| {
error!("binned_json: {e:?}");
let msg = format!("can not parse query: {}", e.msg());
e.add_public_msg(msg)
})?;
@@ -56,6 +57,13 @@ async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Re
.map_err(Error::from)
.map_err(|e| e.add_public_msg(format!("Can not parse query url")))?
};
if req
.uri()
.path_and_query()
.map_or(false, |x| x.as_str().contains("DOERR"))
{
Err(Error::with_msg_no_trace("hidden message").add_public_msg("PublicMessage"))?;
}
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
Ok(binned_json(url, req, node_config).await?)
} else if accept == APP_OCTET {
@@ -86,7 +94,10 @@ impl BinnedHandler {
}
match binned(req, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => Ok(e.to_public_response()),
Err(e) => {
warn!("BinnedHandler handle sees: {e}");
Ok(e.to_public_response())
}
}
}
}

4
httpret/src/api4/status.rs
View File

@@ -44,7 +44,7 @@ impl StatusNodesRecursive {
let res = match res {
Ok(res) => res,
Err(e) => {
let e = Error::from(e);
let e = Error::from(e).add_public_msg("see timeout");
return Ok(crate::bodystream::ToPublicResponse::to_public_response(&e));
}
};
@@ -55,7 +55,7 @@ impl StatusNodesRecursive {
Ok(ret)
}
Err(e) => {
error!("{e}");
error!("StatusNodesRecursive sees: {e}");
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
Ok(ret)
}

36
httpret/src/channel_status.rs
View File

@@ -2,11 +2,18 @@ use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use futures_util::StreamExt;
use http::{Method, Request, Response, StatusCode};
use http::Method;
use http::Request;
use http::Response;
use http::StatusCode;
use hyper::Body;
use items_2::channelevents::ConnStatusEvent;
use netpod::log::*;
use netpod::query::ChannelStateEventsQuery;
use netpod::{FromUrl, NodeConfigCached, ACCEPT_ALL, APP_JSON};
use netpod::FromUrl;
use netpod::NodeConfigCached;
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
use url::Url;
pub struct ConnectionStatusEvents {}
@@ -32,7 +39,7 @@ impl ConnectionStatusEvents {
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelStateEventsQuery::from_url(&url)?;
match self.fetch_data(&q, node_config).await {
@@ -40,8 +47,11 @@ impl ConnectionStatusEvents {
let body = Body::from(serde_json::to_vec(&k)?);
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{:?}", e.public_msg())))?),
Err(e) => {
error!("{e}");
Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{:?}", e.public_msg())))?)
}
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
@@ -61,7 +71,7 @@ impl ConnectionStatusEvents {
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
let scy = scyllaconn::create_scy_session(scyco).await?;
let chconf = dbconn::channelconfig::chconf_from_database(q.channel(), node_config).await?;
let series = chconf.series;
@@ -100,7 +110,7 @@ impl ChannelStatusEvents {
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelStateEventsQuery::from_url(&url)?;
match self.fetch_data(&q, node_config).await {
@@ -108,8 +118,11 @@ impl ChannelStatusEvents {
let body = Body::from(serde_json::to_vec(&k)?);
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{:?}", e.public_msg())))?),
Err(e) => {
error!("{e}");
Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{:?}", e.public_msg())))?)
}
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
@@ -129,13 +142,12 @@ impl ChannelStatusEvents {
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
let scy = scyllaconn::create_scy_session(scyco).await?;
let chconf = dbconn::channelconfig::chconf_from_database(q.channel(), node_config).await?;
let series = chconf.series;
let do_one_before_range = true;
let mut stream =
scyllaconn::status::StatusStreamScylla::new(series, q.range().clone(), do_one_before_range, scy);
scyllaconn::status::StatusStreamScylla::new(chconf.series, q.range().clone(), do_one_before_range, scy);
let mut ret = Vec::new();
while let Some(item) = stream.next().await {
let item = item?;

5
httpret/src/events.rs
View File

@@ -28,7 +28,10 @@ impl EventsHandler {
}
match plain_events(req, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => Ok(e.to_public_response()),
Err(e) => {
error!("EventsHandler sees {e}");
Ok(e.to_public_response())
}
}
}
}

2
httpret/src/proxy.rs
View File

@@ -513,7 +513,7 @@ where
return Ok(res);
}
Err(e) => {
warn!("{e}");
warn!("FT sees: {e}");
let res = crate::bodystream::ToPublicResponse::to_public_response(&e);
return Ok(res);
}

4
httpret/src/proxy/api4.rs
View File

@@ -1,3 +1,5 @@
pub mod caioclookup;
use crate::bodystream::ToPublicResponse;
use crate::err::Error;
use crate::gather::gather_get_json_generic;
@@ -169,7 +171,7 @@ impl StatusNodesRecursive {
Ok(ret)
}
Err(e) => {
error!("{e}");
error!("StatusNodesRecursive sees: {e}");
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
Ok(ret)
}

54
httpret/src/proxy/api4/caioclookup.rs Normal file
View File

@@ -0,0 +1,54 @@
use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use http::Request;
use http::Response;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::ProxyConfig;
pub struct CaIocLookup {}
impl CaIocLookup {
fn path() -> &'static str {
"/api/4/channel-access/search/addr"
}
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == Self::path() {
Some(Self {})
} else {
None
}
}
pub async fn handle(
&self,
req: Request<Body>,
ctx: &ReqCtx,
node_config: &ProxyConfig,
) -> Result<Response<Body>, Error> {
match self.search(req, ctx, node_config).await {
Ok(status) => {
let body = serde_json::to_vec(&status)?;
let ret = response(StatusCode::OK).body(Body::from(body))?;
Ok(ret)
}
Err(e) => {
error!("sees: {e}");
let ret = crate::bodystream::ToPublicResponse::to_public_response(&e);
Ok(ret)
}
}
}
async fn search(
&self,
_req: Request<Body>,
_ctx: &ReqCtx,
_proxy_config: &ProxyConfig,
) -> Result<Option<String>, Error> {
Ok(None)
}
}