daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add status board for post-fetch response status (for py data api 3)

Browse Source
This commit is contained in:
Dominik Werder
2022-03-02 09:55:11 +01:00
parent aa6db72e8e
commit 0b741d187e
7 changed files with 385 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
httpret/src/api1.rs
View File

@@ -9,8 +9,9 @@ use http::{Method, StatusCode};
use hyper::{Body, Client, Request, Response};
use items::{RangeCompletableItem, Sitemty, StreamItem};
use itertools::Itertools;
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::timeunits::SEC;
use netpod::{log::*, ACCEPT_ALL};
use netpod::{ByteSize, Channel, FileIoBufferSize, NanoRange, NodeConfigCached, PerfOpts, Shape, APP_OCTET};
use netpod::{ChannelSearchQuery, ChannelSearchResult, ProxyConfig, APP_JSON};
use parse::channelconfig::{
@@ -21,7 +22,7 @@ use serde_json::Value as JsonValue;
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use std::time::{Duration, Instant};
use url::Url;
pub trait BackendAware {
@@ -500,11 +501,21 @@ pub struct Api1Range {
pub struct Api1Query {
channels: Vec<String>,
range: Api1Range,
// The following are unofficial and not-to-be-used parameters:
// All following parameters are private and not to be used
#[serde(rename = "fileIoBufferSize", default)]
file_io_buffer_size: Option<FileIoBufferSize>,
#[serde(default)]
decompress: bool,
#[serde(rename = "eventsMax", default = "u64_max", skip_serializing_if = "is_u64_max")]
events_max: u64,
}
fn u64_max() -> u64 {
u64::MAX
}
fn is_u64_max(x: &u64) -> bool {
*x == u64::MAX
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -527,6 +538,11 @@ pub struct DataApiPython3DataStream {
config_fut: Option<Pin<Box<dyn Future<Output = Result<Config, Error>> + Send>>>,
file_io_buffer_size: FileIoBufferSize,
do_decompress: bool,
#[allow(unused)]
event_count: u64,
events_max: u64,
status_id: String,
ping_last: Instant,
data_done: bool,
completed: bool,
}
@@ -537,6 +553,8 @@ impl DataApiPython3DataStream {
channels: Vec<Channel>,
file_io_buffer_size: FileIoBufferSize,
do_decompress: bool,
events_max: u64,
status_id: String,
node_config: NodeConfigCached,
) -> Self {
Self {
@@ -548,6 +566,10 @@ impl DataApiPython3DataStream {
config_fut: None,
file_io_buffer_size,
do_decompress,
event_count: 0,
events_max,
status_id,
ping_last: Instant::now(),
data_done: false,
completed: false,
}
@@ -562,9 +584,12 @@ impl DataApiPython3DataStream {
) -> Result<BytesMut, Error> {
let mut d = BytesMut::new();
for i1 in 0..b.tss.len() {
if *count_events < 6 {
const EVIMAX: usize = 6;
if *count_events < EVIMAX {
info!(
"deco len {:?} BE {} scalar-type {:?} shape {:?} comps {:?}",
"ev info {}/{} decomps len {:?} BE {:?} scalar-type {:?} shape {:?} comps {:?}",
*count_events + 1,
EVIMAX,
b.decomps[i1].as_ref().map(|x| x.len()),
b.be[i1],
b.scalar_types[i1],
@@ -657,10 +682,26 @@ impl Stream for DataApiPython3DataStream {
match stream.poll_next_unpin(cx) {
Ready(k) => match k {
Some(k) => match k {
Ok(k) => Ready(Some(Ok(k))),
Ok(k) => {
let n = Instant::now();
if n.duration_since(self.ping_last) >= Duration::from_millis(2000) {
let mut sb = crate::status_board().unwrap();
sb.mark_alive(&self.status_id);
self.ping_last = n;
}
Ready(Some(Ok(k)))
}
Err(e) => {
error!("DataApiPython3DataStream emit error: {e:?}");
self.data_done = true;
Ready(Some(Err(e)))
let mut sb = crate::status_board().unwrap();
sb.add_error(&self.status_id, e);
// TODO format as python data api error frame:
//let mut buf = BytesMut::with_capacity(1024);
//buf.put_slice("".as_bytes());
//Ready(Some(Ok(buf)))
self.data_done = true;
Ready(None)
}
},
None => {
@@ -749,7 +790,12 @@ impl Stream for DataApiPython3DataStream {
}
});
//let _ = Box::new(s) as Box<dyn Stream<Item = Result<BytesMut, Error>> + Unpin>;
self.chan_stream = Some(Box::pin(s.map_err(Error::from)));
let evm = if self.events_max == 0 {
usize::MAX
} else {
self.events_max as usize
};
self.chan_stream = Some(Box::pin(s.map_err(Error::from).take(evm)));
continue;
}
Ready(Err(e)) => {
@@ -778,70 +824,6 @@ impl Stream for DataApiPython3DataStream {
}
}
pub async fn api1_binary_events(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
debug!("api1_binary_events uri: {:?} headers: {:?}", req.uri(), req.headers());
let accept_def = "";
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def))
.to_owned();
let (_head, body) = req.into_parts();
let body_data = hyper::body::to_bytes(body).await?;
let qu: Api1Query = if let Ok(qu) = serde_json::from_slice(&body_data) {
qu
} else {
error!("got body_data: {:?}", String::from_utf8(body_data[..].to_vec()));
return Err(Error::with_msg_no_trace("can not parse query"));
};
debug!("got Api1Query: {:?}", qu);
let beg_date = chrono::DateTime::parse_from_rfc3339(&qu.range.start_date);
let end_date = chrono::DateTime::parse_from_rfc3339(&qu.range.end_date);
let beg_date = beg_date?;
let end_date = end_date?;
debug!("beg_date {:?} end_date {:?}", beg_date, end_date);
//let url = Url::parse(&format!("dummy:{}", req.uri()))?;
//let query = PlainEventsBinaryQuery::from_url(&url)?;
// TODO add stricter check for types, check with client.
if accept == APP_OCTET {}
if false {
let e = Error::with_msg(format!("unexpected Accept: {:?}", accept));
error!("{:?}", e);
return Err(e);
}
let beg_ns = beg_date.timestamp() as u64 * 1000000000 + beg_date.timestamp_subsec_nanos() as u64;
let end_ns = end_date.timestamp() as u64 * 1000000000 + end_date.timestamp_subsec_nanos() as u64;
let range = NanoRange {
beg: beg_ns,
end: end_ns,
};
// TODO use the proper backend name:
let backend = "DUMMY";
let chans = qu
.channels
.iter()
.map(|x| Channel {
backend: backend.into(),
name: x.clone(),
})
.collect();
let file_io_buffer_size = if let Some(k) = qu.file_io_buffer_size {
k
} else {
node_config.node_config.cluster.file_io_buffer_size.clone()
};
let s = DataApiPython3DataStream::new(
range.clone(),
chans,
file_io_buffer_size,
qu.decompress,
node_config.clone(),
);
let ret = response(StatusCode::OK).header("x-daqbuffer-request-id", "dummy");
let ret = ret.body(BodyStream::wrapped(s, format!("api1_binary_events")))?;
return Ok(ret);
}
fn shape_to_api3proto(sh: &Option<Vec<u32>>) -> Vec<u32> {
match sh {
None => vec![],
@@ -856,3 +838,91 @@ fn shape_to_api3proto(sh: &Option<Vec<u32>>) -> Vec<u32> {
}
}
}
pub struct Api1EventsBinaryHandler {}
impl Api1EventsBinaryHandler {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/1/query" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!(
"Api1EventsBinaryHandler::handle uri: {:?} headers: {:?}",
req.uri(),
req.headers()
);
if req.method() != Method::POST {
return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?);
}
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(Ok(ACCEPT_ALL), |k| k.to_str())
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
.to_owned();
let (_head, body) = req.into_parts();
let body_data = hyper::body::to_bytes(body).await?;
let qu: Api1Query = if let Ok(qu) = serde_json::from_slice(&body_data) {
qu
} else {
error!("got body_data: {:?}", String::from_utf8(body_data[..].to_vec()));
return Err(Error::with_msg_no_trace("can not parse query"));
};
info!(
"Api1Query {:?} {} {:?}",
qu.range,
qu.channels.len(),
qu.channels.first()
);
let beg_date = chrono::DateTime::parse_from_rfc3339(&qu.range.start_date);
let end_date = chrono::DateTime::parse_from_rfc3339(&qu.range.end_date);
let beg_date = beg_date?;
let end_date = end_date?;
info!("Api1Query beg_date {:?} end_date {:?}", beg_date, end_date);
//let url = Url::parse(&format!("dummy:{}", req.uri()))?;
//let query = PlainEventsBinaryQuery::from_url(&url)?;
if accept != APP_OCTET && accept != ACCEPT_ALL {
// TODO set the public error code and message and return Err(e).
let e = Error::with_public_msg(format!("Unsupported Accept: {:?}", accept));
error!("{e:?}");
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
let beg = beg_date.timestamp() as u64 * SEC + beg_date.timestamp_subsec_nanos() as u64;
let end = end_date.timestamp() as u64 * SEC + end_date.timestamp_subsec_nanos() as u64;
let range = NanoRange { beg, end };
// TODO check for valid given backend name:
let backend = &node_config.node_config.cluster.backend;
let chans = qu
.channels
.iter()
.map(|x| Channel {
backend: backend.into(),
name: x.clone(),
})
.collect();
let file_io_buffer_size = if let Some(k) = qu.file_io_buffer_size {
k
} else {
node_config.node_config.cluster.file_io_buffer_size.clone()
};
// TODO use a better stream protocol with built-in error delivery.
let status_id = super::status_board()?.new_status_id();
let s = DataApiPython3DataStream::new(
range.clone(),
chans,
file_io_buffer_size,
qu.decompress,
qu.events_max,
status_id.clone(),
node_config.clone(),
);
let ret = response(StatusCode::OK).header("x-daqbuffer-request-id", status_id);
let ret = ret.body(BodyStream::wrapped(s, format!("Api1EventsBinaryHandler")))?;
Ok(ret)
}
}

161
httpret/src/httpret.rs
View File

@@ -31,6 +31,11 @@ use netpod::{ACCEPT_ALL, APP_JSON, APP_JSON_LINES, APP_OCTET};
use nodenet::conn::events_service;
use panic::{AssertUnwindSafe, UnwindSafe};
use pin::Pin;
use serde::Serialize;
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicPtr, Ordering};
use std::sync::{Once, RwLock, RwLockWriteGuard};
use std::time::SystemTime;
use std::{future, net, panic, pin, task};
use task::{Context, Poll};
use tracing::field::Empty;
@@ -42,6 +47,13 @@ fn proxy_mark() -> &'static str {
}
pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
static STATUS_BOARD_INIT: Once = Once::new();
STATUS_BOARD_INIT.call_once(|| {
let b = StatusBoard::new();
let a = RwLock::new(b);
let x = Box::new(a);
STATUS_BOARD.store(Box::into_raw(x), Ordering::SeqCst);
});
let _update_task = if node_config.node_config.cluster.run_map_pulse_task {
Some(UpdateTask::new(node_config.clone()))
} else {
@@ -287,12 +299,8 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/1/query" {
if req.method() == Method::POST {
Ok(api1::api1_binary_events(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if let Some(h) = api1::Api1EventsBinaryHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = evinfo::EventInfoScan::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = pulsemap::IndexFullHttpFunction::handler(&req) {
@@ -479,6 +487,26 @@ impl ToPublicResponse for ::err::Error {
}
}
pub struct StatusBoardAllHandler {}
impl StatusBoardAllHandler {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/status/board/all" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, _req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
use std::ops::Deref;
let sb = status_board().unwrap();
let buf = serde_json::to_vec(sb.deref()).unwrap();
let res = response(StatusCode::OK).body(Body::from(buf))?;
Ok(res)
}
}
async fn binned(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
match binned_inner(req, node_config).await {
Ok(ret) => Ok(ret),
@@ -731,6 +759,127 @@ pub async fn update_search_cache(req: Request<Body>, node_config: &NodeConfigCac
Ok(ret)
}
#[derive(Serialize)]
pub struct StatusBoardEntry {
#[allow(unused)]
#[serde(serialize_with = "instant_serde::ser")]
ts_created: SystemTime,
#[serde(serialize_with = "instant_serde::ser")]
ts_updated: SystemTime,
is_error: bool,
is_ok: bool,
errors: Vec<Error>,
}
mod instant_serde {
use super::*;
use serde::Serializer;
pub fn ser<S: Serializer>(x: &SystemTime, ser: S) -> Result<S::Ok, S::Error> {
let dur = x.duration_since(std::time::UNIX_EPOCH).unwrap();
let dt = chrono::TimeZone::timestamp(&chrono::Utc, dur.as_secs() as i64, dur.subsec_nanos());
let s = dt.format("%Y-%m-%d %H:%M:%S").to_string();
ser.serialize_str(&s)
}
}
impl StatusBoardEntry {
pub fn new() -> Self {
Self {
ts_created: SystemTime::now(),
ts_updated: SystemTime::now(),
is_error: false,
is_ok: false,
errors: vec![],
}
}
}
#[derive(Serialize)]
pub struct StatusBoard {
entries: BTreeMap<String, StatusBoardEntry>,
}
impl StatusBoard {
pub fn new() -> Self {
Self {
entries: BTreeMap::new(),
}
}
pub fn new_status_id(&mut self) -> String {
use std::fs::File;
use std::io::Read;
self.clean();
let mut f = File::open("/dev/urandom").unwrap();
let mut buf = [0; 8];
f.read_exact(&mut buf).unwrap();
let n = u64::from_le_bytes(buf);
let s = format!("{:016x}", n);
self.entries.insert(s.clone(), StatusBoardEntry::new());
s
}
pub fn clean(&mut self) {
if self.entries.len() > 15000 {
let mut tss: Vec<_> = self.entries.values().map(|e| e.ts_updated).collect();
tss.sort_unstable();
let tss = tss;
let tsm = tss[tss.len() / 3];
let a = std::mem::replace(&mut self.entries, BTreeMap::new());
self.entries = a.into_iter().filter(|(_k, v)| v.ts_updated >= tsm).collect();
}
}
pub fn mark_alive(&mut self, status_id: &str) {
match self.entries.get_mut(status_id) {
Some(e) => {
e.ts_updated = SystemTime::now();
}
None => {
error!("can not find status id {}", status_id);
}
}
}
pub fn mark_ok(&mut self, status_id: &str) {
match self.entries.get_mut(status_id) {
Some(e) => {
e.ts_updated = SystemTime::now();
if !e.is_error {
e.is_ok = true;
}
}
None => {
error!("can not find status id {}", status_id);
}
}
}
pub fn add_error(&mut self, status_id: &str, error: Error) {
match self.entries.get_mut(status_id) {
Some(e) => {
e.ts_updated = SystemTime::now();
e.is_error = true;
e.is_ok = false;
e.errors.push(error);
}
None => {
error!("can not find status id {}", status_id);
}
}
}
}
static STATUS_BOARD: AtomicPtr<RwLock<StatusBoard>> = AtomicPtr::new(std::ptr::null_mut());
pub fn status_board() -> Result<RwLockWriteGuard<'static, StatusBoard>, Error> {
let x = unsafe { &*STATUS_BOARD.load(Ordering::SeqCst) }.write();
match x {
Ok(x) => Ok(x),
Err(e) => Err(Error::with_msg(format!("{e:?}"))),
}
}
pub async fn ca_connect_1(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);