daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rustc panics

Browse Source
This commit is contained in:
Dominik Werder
2023-07-18 11:27:39 +02:00
parent 2054f6c23f
commit 91947dec0f
49 changed files with 982 additions and 679 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/httpret/src/api1.rs
+85 -63
View File
@@ -1,14 +1,13 @@
use crate::err::Error;
use crate::gather::gather_get_json_generic;
use crate::gather::SubRes;
use crate::response;
use crate::BodyStream;
use crate::ReqCtx;
use bytes::BufMut;
use bytes::BytesMut;
use disk::eventchunker::EventChunkerConf;
use disk::merge::mergedblobsfromremotes::MergedBlobsFromRemotes;
use disk::raw::conn::make_local_event_blobs_stream;
use err::Error;
use futures_util::FutureExt;
use futures_util::Stream;
use futures_util::StreamExt;
@@ -36,12 +35,14 @@ use netpod::ChannelTypeConfigGen;
use netpod::DiskIoTune;
use netpod::NodeConfigCached;
use netpod::ProxyConfig;
use netpod::ReqCtxArc;
use netpod::SfChFetchInfo;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
use netpod::APP_OCTET;
use netpod::X_DAQBUF_REQID;
use parse::api1_parse::Api1ByteOrder;
use parse::api1_parse::Api1ChannelHeader;
use query::api4::events::EventsSubQuery;
@@ -529,15 +530,16 @@ pub struct DataApiPython3DataStream {
channels: VecDeque<ChannelTypeConfigGen>,
settings: EventsSubQuerySettings,
current_channel: Option<ChannelTypeConfigGen>,
current_fetch_info: Option<SfChFetchInfo>,
node_config: NodeConfigCached,
chan_stream: Option<Pin<Box<dyn Stream<Item = Result<BytesMut, Error>> + Send>>>,
chan_stream: Option<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<Option<ChannelTypeConfigGen>, Error>> + Send>>>,
disk_io_tune: DiskIoTune,
do_decompress: bool,
#[allow(unused)]
event_count: u64,
event_count: usize,
events_max: u64,
status_id: String,
header_out: bool,
reqctx: ReqCtxArc,
ping_last: Instant,
data_done: bool,
completed: bool,
@@ -551,7 +553,7 @@ impl DataApiPython3DataStream {
disk_io_tune: DiskIoTune,
do_decompress: bool,
events_max: u64,
status_id: String,
reqctx: ReqCtxArc,
node_config: NodeConfigCached,
) -> Self {
Self {
@@ -559,6 +561,7 @@ impl DataApiPython3DataStream {
channels: channels.into_iter().collect(),
settings,
current_channel: None,
current_fetch_info: None,
node_config,
chan_stream: None,
config_fut: None,
@@ -566,7 +569,8 @@ impl DataApiPython3DataStream {
do_decompress,
event_count: 0,
events_max,
status_id,
header_out: false,
reqctx,
ping_last: Instant::now(),
data_done: false,
completed: false,
@@ -577,6 +581,7 @@ impl DataApiPython3DataStream {
b: EventFull,
channel: &ChannelTypeConfigGen,
fetch_info: &SfChFetchInfo,
do_decompress: bool,
header_out: &mut bool,
count_events: &mut usize,
) -> Result<BytesMut, Error> {
@@ -586,10 +591,10 @@ impl DataApiPython3DataStream {
const EVIMAX: usize = 6;
if *count_events < EVIMAX {
debug!(
"ev info {}/{} decomps len {:?} BE {:?} scalar-type {:?} shape {:?} comps {:?}",
"ev info {}/{} bloblen {:?} BE {:?} scalar-type {:?} shape {:?} comps {:?}",
*count_events + 1,
EVIMAX,
b.decomps[i1].as_ref().map(|x| x.len()),
b.blobs[i1].len(),
b.be[i1],
b.scalar_types[i1],
b.shapes[i1],
@@ -618,26 +623,38 @@ impl DataApiPython3DataStream {
b.comps.get(i1).map(|x| x.clone()).unwrap(),
);
let h = serde_json::to_string(&head)?;
info!("sending channel header {}", h);
debug!("sending channel header {}", h);
let l1 = 1 + h.as_bytes().len() as u32;
d.put_u32(l1);
d.put_u8(0);
info!("header frame byte len {}", 4 + 1 + h.as_bytes().len());
debug!("header frame byte len {}", 4 + 1 + h.as_bytes().len());
d.extend_from_slice(h.as_bytes());
d.put_u32(l1);
*header_out = true;
}
match &b.shapes[i1] {
_ => {
let empty_blob = Vec::new();
let blob = b.blobs[i1].as_ref().unwrap_or(&empty_blob);
let l1 = 17 + blob.len() as u32;
d.put_u32(l1);
d.put_u8(1);
d.put_u64(b.tss[i1]);
d.put_u64(b.pulses[i1]);
d.put_slice(&blob);
d.put_u32(l1);
if do_decompress {
let blob = b
.data_decompressed(i1, fetch_info.scalar_type(), fetch_info.shape())
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let l1 = 17 + blob.len() as u32;
d.put_u32(l1);
d.put_u8(1);
d.put_u64(b.tss[i1]);
d.put_u64(b.pulses[i1]);
d.put_slice(&blob);
d.put_u32(l1);
} else {
let blob = b.data_raw(i1);
let l1 = 17 + blob.len() as u32;
d.put_u32(l1);
d.put_u8(1);
d.put_u64(b.tss[i1]);
d.put_u64(b.pulses[i1]);
d.put_slice(&blob);
d.put_u32(l1);
}
}
}
*count_events += 1;
@@ -645,23 +662,42 @@ impl DataApiPython3DataStream {
Ok(d)
}
fn handle_chan_stream_ready(&mut self, item: Result<BytesMut, Error>) -> Option<Result<BytesMut, Error>> {
fn handle_chan_stream_ready(&mut self, item: Sitemty<EventFull>) -> Option<Result<BytesMut, Error>> {
match item {
Ok(k) => {
let n = Instant::now();
if n.duration_since(self.ping_last) >= Duration::from_millis(2000) {
let mut sb = crate::status_board().unwrap();
sb.mark_alive(&self.status_id);
sb.mark_alive(self.reqctx.reqid());
self.ping_last = n;
}
Some(Ok(k))
match k {
StreamItem::DataItem(k) => match k {
RangeCompletableItem::RangeComplete => todo!(),
RangeCompletableItem::Data(k) => {
let item = Self::convert_item(
k,
self.current_channel.as_ref().unwrap(),
self.current_fetch_info.as_ref().unwrap(),
self.do_decompress,
&mut self.header_out,
&mut self.event_count,
)?;
todo!()
}
},
StreamItem::Log(k) => todo!(),
StreamItem::Stats(k) => todo!(),
}
}
Err(e) => {
error!("DataApiPython3DataStream emit error: {e:?}");
self.chan_stream = None;
self.current_channel = None;
self.current_fetch_info = None;
self.data_done = true;
let mut sb = crate::status_board().unwrap();
sb.add_error(&self.status_id, e);
sb.add_error(self.reqctx.reqid(), e);
if false {
// TODO format as python data api error frame:
let mut buf = BytesMut::with_capacity(1024);
@@ -682,14 +718,14 @@ impl DataApiPython3DataStream {
self.range.clone().into(),
TransformQuery::for_event_blobs(),
);
let subq = EventsSubQuery::from_parts(select, self.settings.clone());
let subq = EventsSubQuery::from_parts(select, self.settings.clone(), self.reqctx.reqid().into());
let one_before = subq.transform().need_one_before_range();
debug!("query for event blobs retrieval subq {subq:?}");
// TODO important TODO
debug!("TODO fix magic inmem_bufcap");
debug!("TODO add timeout option to data api3 download");
// TODO is this a good to place decide this?
let s = if self.node_config.node_config.cluster.is_central_storage {
let stream = if self.node_config.node_config.cluster.is_central_storage {
info!("Set up central storage stream");
// TODO pull up this config
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
@@ -697,9 +733,9 @@ impl DataApiPython3DataStream {
self.range.clone(),
fetch_info.clone(),
one_before,
self.do_decompress,
event_chunker_conf,
self.disk_io_tune.clone(),
self.reqctx.clone(),
&self.node_config,
)?;
Box::pin(s) as Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>
@@ -708,33 +744,13 @@ impl DataApiPython3DataStream {
let s = MergedBlobsFromRemotes::new(subq, self.node_config.node_config.cluster.clone());
Box::pin(s) as Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>
};
let s = s.map({
let mut header_out = false;
let mut count_events = 0;
let channel = self.current_channel.clone().unwrap();
move |b| {
let ret = match b {
Ok(b) => {
let f = match b {
StreamItem::DataItem(RangeCompletableItem::Data(b)) => {
Self::convert_item(b, &channel, &fetch_info, &mut header_out, &mut count_events)?
}
_ => BytesMut::new(),
};
Ok(f)
}
Err(e) => Err(e),
};
ret
}
});
//let _ = Box::new(s) as Box<dyn Stream<Item = Result<BytesMut, Error>> + Unpin>;
let evm = if self.events_max == 0 {
usize::MAX
} else {
self.events_max as usize
};
self.chan_stream = Some(Box::pin(s.map_err(Error::from).take(evm)));
self.chan_stream = Some(Box::pin(stream));
self.current_fetch_info = Some(fetch_info);
Ok(())
}
}
@@ -807,9 +823,9 @@ impl Stream for DataApiPython3DataStream {
{
let n = Instant::now();
let mut sb = crate::status_board().unwrap();
sb.mark_alive(&self.status_id);
sb.mark_alive(self.reqctx.reqid());
self.ping_last = n;
sb.mark_ok(&self.status_id);
sb.mark_ok(self.reqctx.reqid());
}
continue;
}
@@ -864,7 +880,7 @@ impl Api1EventsBinaryHandler {
.to_owned();
let body_data = hyper::body::to_bytes(body).await?;
if body_data.len() < 1024 * 2 && body_data.first() == Some(&"{".as_bytes()[0]) {
info!("request body_data string: {}", String::from_utf8_lossy(&body_data));
debug!("request body_data string: {}", String::from_utf8_lossy(&body_data));
}
let qu = match serde_json::from_slice::<Api1Query>(&body_data) {
Ok(mut qu) => {
@@ -879,6 +895,8 @@ impl Api1EventsBinaryHandler {
return Err(Error::with_msg_no_trace("can not parse query"));
}
};
let reqid = super::status_board()?.new_status_id();
let reqctx = netpod::ReqCtx::new(reqid);
let span = if qu.log_level() == "trace" {
debug!("enable trace for handler");
tracing::span!(tracing::Level::TRACE, "log_span_trace")
@@ -888,8 +906,10 @@ impl Api1EventsBinaryHandler {
} else {
tracing::Span::none()
};
self.handle_for_query(qu, accept, span.clone(), node_config)
let reqidspan = tracing::info_span!("api1query", reqid = reqctx.reqid());
self.handle_for_query(qu, accept, &reqctx, span.clone(), reqidspan.clone(), node_config)
.instrument(span)
.instrument(reqidspan)
.await
}
@@ -897,12 +917,14 @@ impl Api1EventsBinaryHandler {
&self,
qu: Api1Query,
accept: String,
reqctx: &ReqCtxArc,
span: tracing::Span,
reqidspan: tracing::Span,
ncc: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let self_name = any::type_name::<Self>();
// TODO this should go to usage statistics:
info!(
debug!(
"{self_name} {:?} {} {:?}",
qu.range(),
qu.channels().len(),
@@ -925,24 +947,24 @@ impl Api1EventsBinaryHandler {
let ts1 = Instant::now();
let mut chans = Vec::new();
for ch in qu.channels() {
info!("try to find config quorum for {ch:?}");
debug!("try to find config quorum for {ch:?}");
let ch = SfDbChannel::from_name(backend, ch.name());
let ch_conf =
nodenet::configquorum::find_config_basics_quorum(ch.clone(), range.clone().into(), ncc).await?;
match ch_conf {
Some(x) => {
debug!("found quorum {ch:?} {x:?}");
chans.push(x);
}
None => {
// TODO count in request ctx.
error!("no config quorum found for {ch:?}");
}
}
}
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1).as_millis();
info!("{self_name} configs fetched in {} ms", dt);
// TODO use a better stream protocol with built-in error delivery.
let status_id = super::status_board()?.new_status_id();
debug!("{self_name} {} configs fetched in {} ms", chans.len(), dt);
let s = DataApiPython3DataStream::new(
range.clone(),
chans,
@@ -951,12 +973,12 @@ impl Api1EventsBinaryHandler {
DiskIoTune::default(),
qu.decompress(),
qu.events_max().unwrap_or(u64::MAX),
status_id.clone(),
reqctx.clone(),
ncc.clone(),
);
let s = s.instrument(span);
let body = BodyStream::wrapped(s, format!("Api1EventsBinaryHandler"));
let ret = response(StatusCode::OK).header("x-daqbuffer-request-id", status_id);
let s = s.instrument(span).instrument(reqidspan);
let body = Body::wrap_stream(s);
let ret = response(StatusCode::OK).header(X_DAQBUF_REQID, reqctx.reqid());
let ret = ret.body(body)?;
Ok(ret)
} else {
crates/httpret/src/api4/binned.rs
+4 -3
View File
@@ -1,8 +1,8 @@
use crate::bodystream::response;
use crate::bodystream::ToPublicResponse;
use crate::channelconfig::ch_conf_from_binned;
use crate::err::Error;
use crate::response_err;
use err::Error;
use http::Method;
use http::Request;
use http::Response;
@@ -20,7 +20,8 @@ use tracing::Instrument;
use url::Url;
async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
debug!("httpret plain_events_json req: {:?}", req);
debug!("{:?}", req);
let reqid = crate::status_board()?.new_status_id();
let (_head, _body) = req.into_parts();
let query = BinnedQuery::from_url(&url).map_err(|e| {
error!("binned_json: {e:?}");
@@ -41,7 +42,7 @@ async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCache
span1.in_scope(|| {
debug!("begin");
});
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, node_config.node_config.cluster.clone())
let item = streams::timebinnedjson::timebinned_json(query, ch_conf, reqid, node_config.node_config.cluster.clone())
.instrument(span1)
.await?;
let buf = serde_json::to_vec(&item)?;
crates/httpret/src/api4/databuffer_tools.rs
+223 -21
View File
@@ -1,33 +1,40 @@
use crate::bodystream::response;
use crate::bodystream::BodyStream;
use crate::response_err;
use async_channel::Receiver;
use async_channel::Sender;
use bytes::Bytes;
use err::thiserror;
use err::ThisError;
use err::ToPublicError;
use futures_util::Stream;
use futures_util::StreamExt;
use http::Method;
use http::Request;
use http::Response;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::Node;
use netpod::NodeConfigCached;
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
use serde::Serialize;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[derive(Debug, ThisError)]
pub enum FindActiveError {
#[error("HttpBadAccept")]
HttpBadAccept,
#[error("HttpBadUrl")]
HttpBadUrl,
#[error("{0}")]
#[error("Error({0})")]
Error(Box<dyn ToPublicError>),
#[error("{0}")]
#[error("UrlError({0})")]
UrlError(#[from] url::ParseError),
#[error("InternalError")]
InternalError,
IO(#[from] std::io::Error),
}
impl ToPublicError for FindActiveError {
@@ -36,8 +43,9 @@ impl ToPublicError for FindActiveError {
FindActiveError::HttpBadAccept => format!("{self}"),
FindActiveError::HttpBadUrl => format!("{self}"),
FindActiveError::Error(e) => e.to_public_error(),
FindActiveError::UrlError(e) => format!("can not parse url: {e}"),
FindActiveError::UrlError(_) => format!("{self}"),
FindActiveError::InternalError => format!("{self}"),
FindActiveError::IO(_) => format!("{self}"),
}
}
}
@@ -46,7 +54,7 @@ pub struct FindActiveHandler {}
impl FindActiveHandler {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/tools/databuffer/findActive" {
if req.uri().path() == "/api/4/tool/sfdatabuffer/find/channel/active" {
Some(Self {})
} else {
None
@@ -83,28 +91,222 @@ impl FindActiveHandler {
};
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
type _A = netpod::BodyStream;
Ok(Response::builder()
.status(StatusCode::OK)
.body(BodyStream::wrapped(Box::pin(DummyStream::new()), "find_active".into()))
.map_err(|_| FindActiveError::InternalError)?)
let stream = FindActiveStream::new(40, 2, ncc);
let stream = stream.chain(FindActiveStream::new(40, 3, ncc));
let stream = stream
.map(|item| match item {
Ok(item) => {
let mut s = serde_json::to_vec(&item).unwrap();
s.push(0x0a);
s
}
Err(e) => {
error!("ERROR in http body stream after headers: {e}");
Vec::new()
}
})
.map(|x| Ok::<_, String>(Bytes::from(x)));
let body = Body::wrap_stream(Box::pin(stream));
Ok(Response::builder().status(StatusCode::OK).body(body).unwrap())
} else {
Err(FindActiveError::HttpBadAccept)
}
}
}
struct DummyStream {}
#[derive(Debug, Serialize)]
struct ActiveChannelDesc {
ks: u32,
name: String,
totlen: u64,
}
impl DummyStream {
pub fn new() -> Self {
todo!()
async fn sum_dir_contents(path: PathBuf) -> Result<u64, FindActiveError> {
let mut sum = 0;
let mut dir_stream = tokio::fs::read_dir(path).await?;
loop {
match dir_stream.next_entry().await? {
Some(x) => {
if x.file_name().to_string_lossy().starts_with("..") {
debug!("INCONVENIENT: {x:?}");
} else if x.file_type().await.unwrap().is_dir() {
let mut dir_stream_2 = tokio::fs::read_dir(x.path()).await?;
loop {
match dir_stream_2.next_entry().await? {
Some(x) => {
let md = x.metadata().await?;
sum += md.len();
}
None => break,
}
}
} else {
error!("unexpected file: {:?}", x.file_name());
sum += x.metadata().await?.len();
}
}
None => break,
}
}
Ok(sum)
}
struct XorShift32 {
state: u32,
}
impl XorShift32 {
fn new(state: u32) -> Self {
Self { state }
}
fn next(&mut self) -> u32 {
let mut x = self.state;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
self.state = x;
x
}
}
impl Stream for DummyStream {
type Item = Result<Bytes, crate::err::Error>;
async fn find_active_inner(
max: usize,
ks: u32,
splits: &[u64],
node: Node,
tx: Sender<Result<ActiveChannelDesc, FindActiveError>>,
) -> Result<(), FindActiveError> {
let mut count = 0;
let now_sec = std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs();
let mut rng = XorShift32::new(now_sec as u32);
for _ in 0..64 {
rng.next();
}
let tb_exp = now_sec / 60 / 60 / 24;
let re_tb = regex::Regex::new(r"(0000\d{15})").unwrap();
let path = disk::paths::datapath_for_keyspace(ks, &node);
let mut dir_stream = tokio::fs::read_dir(path).await?;
let mut channel_dirs = Vec::new();
loop {
let x = dir_stream.next_entry().await?;
match x {
Some(x) => {
if x.file_name().to_string_lossy().starts_with(".") {
debug!("INCONVENIENT: {x:?}");
} else if x.file_name().to_string_lossy().starts_with("..") {
debug!("INCONVENIENT: {x:?}");
} else {
channel_dirs.push((rng.next(), x));
}
}
None => break,
}
}
channel_dirs.sort_by_key(|x| x.0);
let channel_dirs = channel_dirs;
// TODO randomize channel list using given seed
'outer: for (_, chdir) in channel_dirs {
let ft = chdir.file_type().await?;
if ft.is_dir() {
let mut dir_stream = tokio::fs::read_dir(chdir.path())
.await
.map_err(|e| FindActiveError::IO(e))?;
loop {
match dir_stream.next_entry().await? {
Some(e) => {
let x = e.file_name();
let s = x.to_string_lossy();
if let Some(_) = re_tb.captures(&s) {
let chn1 = chdir.file_name();
let chname = chn1.to_string_lossy();
// debug!("match: {m:?}");
// TODO bin-size depends on channel config
match s.parse::<u64>() {
Ok(x) => {
if x == tb_exp {
// debug!("matching tb {}", chname);
let sum = sum_dir_contents(e.path()).await?;
if sum > 1024 * 1024 * 10 {
// debug!("sizable content: {sum}");
let x = ActiveChannelDesc {
ks,
name: chname.into(),
totlen: sum,
};
tx.send(Ok(x)).await;
count += 1;
if count >= max {
break 'outer;
}
}
}
}
Err(_) => {}
}
} else {
// debug!("no match");
}
}
None => break,
}
}
} else {
error!("unexpected file {chdir:?}");
}
}
Ok(())
}
fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context) -> std::task::Poll<Option<Self::Item>> {
todo!()
async fn find_active(
max: usize,
ks: u32,
splits: Vec<u64>,
node: Node,
tx: Sender<Result<ActiveChannelDesc, FindActiveError>>,
) {
let tx2 = tx.clone();
match find_active_inner(max, ks, &splits, node, tx).await {
Ok(x) => x,
Err(e) => {
tx2.send(Err(e)).await;
return;
}
}
}
struct FindActiveStream {
rx: Receiver<Result<ActiveChannelDesc, FindActiveError>>,
}
impl FindActiveStream {
pub fn new(max: usize, ks: u32, ncc: &NodeConfigCached) -> Self {
let (tx, rx) = async_channel::bounded(4);
let splits = ncc
.node
.sf_databuffer
.as_ref()
.unwrap()
.splits
.as_ref()
.map_or(Vec::new(), Clone::clone);
let _jh = taskrun::spawn(find_active(max, ks, splits, ncc.node.clone(), tx));
Self { rx }
}
}
impl Stream for FindActiveStream {
type Item = Result<ActiveChannelDesc, FindActiveError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
match self.rx.poll_next_unpin(cx) {
Ready(Some(item)) => Ready(Some(item)),
Ready(None) => Ready(None),
Pending => Pending,
}
}
}
crates/httpret/src/api4/events.rs
+6 -8
View File
@@ -1,9 +1,8 @@
use crate::channelconfig::chconf_from_events_v1;
use crate::err::Error;
use crate::response;
use crate::response_err;
use crate::BodyStream;
use crate::ToPublicResponse;
use err::Error;
use futures_util::stream;
use futures_util::TryStreamExt;
use http::Method;
@@ -72,15 +71,12 @@ async fn plain_events_binary(
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
debug!("plain_events_binary req: {:?}", req);
debug!("{:?}", req);
let query = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
let ch_conf = chconf_from_events_v1(&query, node_config).await?;
info!("plain_events_binary chconf_from_events_v1: {ch_conf:?}");
let s = stream::iter([Ok::<_, Error>(String::from("TODO_PREBINNED_BINARY_STREAM"))]);
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
s.map_err(Error::from),
format!("plain_events_binary"),
))?;
let ret = response(StatusCode::OK).body(Body::wrap_stream(s.map_err(Error::from)))?;
Ok(ret)
}
@@ -89,6 +85,7 @@ async fn plain_events_json(
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let reqid = crate::status_board()?.new_status_id();
info!("plain_events_json req: {:?}", req);
let (_head, _body) = req.into_parts();
let query = PlainEventsQuery::from_url(&url)?;
@@ -99,7 +96,8 @@ async fn plain_events_json(
.map_err(Error::from)?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_json chconf_from_events_v1: {ch_conf:?}");
let item = streams::plaineventsjson::plain_events_json(&query, ch_conf, &node_config.node_config.cluster).await;
let item =
streams::plaineventsjson::plain_events_json(&query, ch_conf, reqid, &node_config.node_config.cluster).await;
let item = match item {
Ok(item) => item,
Err(e) => {
crates/httpret/src/api4/search.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::bodystream::response;
use crate::bodystream::ToPublicResponse;
use crate::Error;
use err::Error;
use http::Method;
use http::Request;
use http::Response;
crates/httpret/src/api4/status.rs
+5 -5
View File
@@ -1,6 +1,6 @@
use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use crate::RetrievalError;
use http::Request;
use http::Response;
use http::StatusCode;
@@ -14,7 +14,7 @@ use std::collections::VecDeque;
use std::time::Duration;
#[allow(unused)]
async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, Error> {
async fn table_sizes(node_config: &NodeConfigCached) -> Result<TableSizes, RetrievalError> {
let ret = dbconn::table_sizes(node_config).await?;
Ok(ret)
}
@@ -39,12 +39,12 @@ impl StatusNodesRecursive {
req: Request<Body>,
ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let res = tokio::time::timeout(Duration::from_millis(1200), self.status(req, ctx, node_config)).await;
let res = match res {
Ok(res) => res,
Err(e) => {
let e = Error::from(e).add_public_msg("see timeout");
let e = RetrievalError::from(e).add_public_msg("see timeout");
return Ok(crate::bodystream::ToPublicResponse::to_public_response(&e));
}
};
@@ -67,7 +67,7 @@ impl StatusNodesRecursive {
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<NodeStatus, Error> {
) -> Result<NodeStatus, RetrievalError> {
let (_head, _body) = req.into_parts();
let archiver_appliance_status = match node_config.node.archiver_appliance.as_ref() {
Some(k) => {
crates/httpret/src/bodystream.rs
+6 -88
View File
@@ -1,16 +1,14 @@
use crate::err::Error;
use bytes::Bytes;
use futures_util::{Stream, StreamExt};
use err::Error;
use futures_util::StreamExt;
use http::HeaderMap;
use http::{Response, StatusCode};
use http::Response;
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::APP_JSON;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::task::{Context, Poll};
use tracing::field::Empty;
use tracing::{span, Level};
use std::task::Context;
use std::task::Poll;
pub fn response<T>(status: T) -> http::response::Builder
where
@@ -20,58 +18,6 @@ where
Response::builder().status(status)
}
pub struct BodyStream<S> {
inp: S,
desc: String,
}
impl<S, I> BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin + Send + 'static,
I: Into<Bytes> + Sized + 'static,
{
pub fn new(inp: S, desc: String) -> Self {
Self { inp, desc }
}
pub fn wrapped(inp: S, desc: String) -> Body {
Body::wrap_stream(Self::new(inp, desc))
}
}
impl<S, I> Stream for BodyStream<S>
where
S: Stream<Item = Result<I, Error>> + Unpin,
I: Into<Bytes> + Sized,
{
type Item = Result<I, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let span1 = span!(Level::INFO, "httpret::BodyStream", desc = Empty);
span1.record("desc", &self.desc.as_str());
span1.in_scope(|| {
use Poll::*;
let t = std::panic::catch_unwind(AssertUnwindSafe(|| self.inp.poll_next_unpin(cx)));
match t {
Ok(r) => match r {
Ready(Some(Ok(k))) => Ready(Some(Ok(k))),
Ready(Some(Err(e))) => {
error!("body stream error: {e:?}");
Ready(Some(Err(Error::from(e))))
}
Ready(None) => Ready(None),
Pending => Pending,
},
Err(e) => {
error!("panic caught in httpret::BodyStream: {e:?}");
let e = Error::with_msg(format!("panic caught in httpret::BodyStream: {e:?}"));
Ready(Some(Err(e)))
}
}
})
}
}
pub trait ToPublicResponse {
fn to_public_response(&self) -> Response<Body>;
}
@@ -82,34 +28,6 @@ impl ToPublicResponse for Error {
}
}
impl ToPublicResponse for ::err::Error {
fn to_public_response(&self) -> Response<Body> {
use err::Reason;
let e = self.to_public_error();
let status = match e.reason() {
Some(Reason::BadRequest) => StatusCode::BAD_REQUEST,
Some(Reason::InternalError) => StatusCode::INTERNAL_SERVER_ERROR,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
let msg = match serde_json::to_string(&e) {
Ok(s) => s,
Err(_) => "can not serialize error".into(),
};
match response(status)
.header(http::header::ACCEPT, APP_JSON)
.body(Body::from(msg))
{
Ok(res) => res,
Err(e) => {
error!("can not generate http error response {e:?}");
let mut res = Response::new(Body::default());
*res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
res
}
}
}
}
struct BodyStreamWrap(netpod::BodyStream);
impl hyper::body::HttpBody for BodyStreamWrap {
crates/httpret/src/channel_status.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use err::Error;
use futures_util::StreamExt;
use http::Method;
use http::Request;
crates/httpret/src/channelconfig.rs
+5 -5
View File
@@ -1,7 +1,7 @@
use crate::err::Error;
use crate::response;
use crate::ToPublicResponse;
use dbconn::create_connection;
use err::Error;
use futures_util::StreamExt;
use http::Method;
use http::Request;
@@ -22,6 +22,7 @@ use netpod::SfDbChannel;
use netpod::Shape;
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
use nodenet::configquorum::find_config_basics_quorum;
use query::api4::binned::BinnedQuery;
use query::api4::events::PlainEventsQuery;
use scylla::frame::response::cql_to_rust::FromRowError as ScyFromRowError;
@@ -37,7 +38,7 @@ pub async fn chconf_from_events_v1(
q: &PlainEventsQuery,
ncc: &NodeConfigCached,
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
let ret = find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
Ok(ret)
}
@@ -45,8 +46,7 @@ pub async fn chconf_from_prebinned(
q: &PreBinnedQuery,
ncc: &NodeConfigCached,
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret =
nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.patch().patch_range(), ncc).await?;
let ret = find_config_basics_quorum(q.channel().clone(), q.patch().patch_range(), ncc).await?;
Ok(ret)
}
@@ -54,7 +54,7 @@ pub async fn ch_conf_from_binned(
q: &BinnedQuery,
ncc: &NodeConfigCached,
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
let ret = find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
Ok(ret)
}
crates/httpret/src/download.rs
+21 -7
View File
@@ -1,10 +1,16 @@
use crate::err::Error;
use crate::response;
use crate::RetrievalError;
use futures_util::TryStreamExt;
use http::{Method, StatusCode};
use hyper::{Body, Request, Response};
use http::Method;
use http::StatusCode;
use hyper::Body;
use hyper::Request;
use hyper::Response;
use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::{get_url_query_pairs, DiskIoTune, FromUrl, NodeConfigCached};
use netpod::DiskIoTune;
use netpod::FromUrl;
use netpod::NodeConfigCached;
use url::Url;
#[derive(Clone, Debug)]
@@ -61,7 +67,11 @@ impl DownloadHandler {
}
}
pub async fn get(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
pub async fn get(
&self,
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let p2 = &head.uri.path()[Self::path_prefix().len()..];
let base = match &node_config.node.sf_databuffer {
@@ -74,11 +84,15 @@ impl DownloadHandler {
let pp = base.join(p2);
info!("Try to open {pp:?}");
let file = tokio::fs::OpenOptions::new().read(true).open(&pp).await?;
let s = disk::file_content_stream(pp, file, query.disk_io_tune.clone()).map_ok(|x| x.into_buf());
let s = disk::file_content_stream(pp, file, query.disk_io_tune.clone(), "download").map_ok(|x| x.into_buf());
Ok(response(StatusCode::OK).body(Body::wrap_stream(s))?)
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
pub async fn handle(
&self,
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
if req.method() == Method::GET {
self.get(req, node_config).await
} else {
crates/httpret/src/gather.rs
+31 -18
View File
@@ -1,12 +1,19 @@
use crate::err::Error;
use crate::response;
use futures_util::{select, FutureExt};
use http::{Method, StatusCode};
use hyper::{Body, Client, Request, Response};
use crate::RetrievalError;
use futures_util::select;
use futures_util::FutureExt;
use http::Method;
use http::StatusCode;
use hyper::Body;
use hyper::Client;
use hyper::Request;
use hyper::Response;
use netpod::log::*;
use netpod::Node;
use netpod::NodeConfigCached;
use netpod::APP_JSON;
use netpod::{Node, NodeConfigCached};
use serde::{Deserialize, Serialize};
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
use std::future::Future;
use std::pin::Pin;
@@ -26,7 +33,7 @@ struct GatherHost {
inst: String,
}
async fn process_answer(res: Response<Body>) -> Result<JsonValue, Error> {
async fn process_answer(res: Response<Body>) -> Result<JsonValue, RetrievalError> {
let (pre, mut body) = res.into_parts();
if pre.status != StatusCode::OK {
use hyper::body::HttpBody;
@@ -48,11 +55,14 @@ async fn process_answer(res: Response<Body>) -> Result<JsonValue, Error> {
Ok(k) => k,
Err(_e) => JsonValue::String(String::from_utf8(body_all.to_vec())?),
};
Ok::<_, Error>(val)
Ok::<_, RetrievalError>(val)
}
}
pub async fn unused_gather_json_from_hosts(req: Request<Body>, pathpre: &str) -> Result<Response<Body>, Error> {
pub async fn unused_gather_json_from_hosts(
req: Request<Body>,
pathpre: &str,
) -> Result<Response<Body>, RetrievalError> {
let (part_head, part_body) = req.into_parts();
let bodyslice = hyper::body::to_bytes(part_body).await?;
let gather_from: GatherFrom = serde_json::from_slice(&bodyslice)?;
@@ -72,7 +82,7 @@ pub async fn unused_gather_json_from_hosts(req: Request<Body>, pathpre: &str) ->
let task = tokio::spawn(async move {
select! {
_ = sleep(Duration::from_millis(1500)).fuse() => {
Err(Error::with_msg("timeout"))
Err(RetrievalError::with_msg("timeout"))
}
res = Client::new().request(req?).fuse() => Ok(process_answer(res?).await?)
}
@@ -105,7 +115,10 @@ pub async fn unused_gather_json_from_hosts(req: Request<Body>, pathpre: &str) ->
Ok(res)
}
pub async fn gather_get_json(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
pub async fn gather_get_json(
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
let (head, body) = req.into_parts();
let _bodyslice = hyper::body::to_bytes(body).await?;
let pathpre = "/api/4/gather/";
@@ -123,7 +136,7 @@ pub async fn gather_get_json(req: Request<Body>, node_config: &NodeConfigCached)
let task = tokio::spawn(async move {
select! {
_ = sleep(Duration::from_millis(1500)).fuse() => {
Err(Error::with_msg("timeout"))
Err(RetrievalError::with_msg("timeout"))
}
res = Client::new().request(req?).fuse() => Ok(process_answer(res?).await?)
}
@@ -181,23 +194,23 @@ pub async fn gather_get_json_generic<SM, NT, FT, OUT>(
// TODO use deadline instead.
// TODO Wait a bit longer compared to remote to receive partial results.
timeout: Duration,
) -> Result<OUT, Error>
) -> Result<OUT, RetrievalError>
where
SM: Send + 'static,
NT: Fn(String, Response<Body>) -> Pin<Box<dyn Future<Output = Result<SubRes<SM>, Error>> + Send>>
NT: Fn(String, Response<Body>) -> Pin<Box<dyn Future<Output = Result<SubRes<SM>, RetrievalError>> + Send>>
+ Send
+ Sync
+ Copy
+ 'static,
FT: Fn(Vec<(Tag, Result<SubRes<SM>, Error>)>) -> Result<OUT, Error>,
FT: Fn(Vec<(Tag, Result<SubRes<SM>, RetrievalError>)>) -> Result<OUT, RetrievalError>,
{
// TODO remove magic constant
let extra_timeout = Duration::from_millis(3000);
if urls.len() != bodies.len() {
return Err(Error::with_msg_no_trace("unequal numbers of urls and bodies"));
return Err(RetrievalError::TextError(format!("unequal numbers of urls and bodies")));
}
if urls.len() != tags.len() {
return Err(Error::with_msg_no_trace("unequal numbers of urls and tags"));
return Err(RetrievalError::TextError(format!("unequal numbers of urls and tags")));
}
let spawned: Vec<_> = urls
.into_iter()
@@ -227,7 +240,7 @@ where
select! {
_ = sleep(timeout + extra_timeout).fuse() => {
error!("PROXY TIMEOUT");
Err(Error::with_msg_no_trace("timeout"))
Err(RetrievalError::TextError(format!("timeout")))
}
res = {
let client = Client::new();
crates/httpret/src/httpret.rs
+86 -37
View File
@@ -4,19 +4,18 @@ pub mod bodystream;
pub mod channel_status;
pub mod channelconfig;
pub mod download;
pub mod err;
pub mod gather;
pub mod prometheus;
pub mod proxy;
pub mod pulsemap;
pub mod settings;
use self::bodystream::BodyStream;
use self::bodystream::ToPublicResponse;
use crate::bodystream::response;
use crate::err::Error;
use crate::gather::gather_get_json;
use crate::pulsemap::UpdateTask;
use err::thiserror;
use err::ThisError;
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::StreamExt;
@@ -41,6 +40,7 @@ use nodenet::conn::events_service;
use panic::AssertUnwindSafe;
use panic::UnwindSafe;
use pin::Pin;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
use std::net;
@@ -59,7 +59,37 @@ use task::Poll;
pub const PSI_DAQBUFFER_SERVICE_MARK: &'static str = "PSI-Daqbuffer-Service-Mark";
pub const PSI_DAQBUFFER_SEEN_URL: &'static str = "PSI-Daqbuffer-Seen-Url";
pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
#[derive(Debug, ThisError, Serialize, Deserialize)]
pub enum RetrievalError {
Error(#[from] err::Error),
TextError(String),
#[serde(skip)]
Hyper(#[from] hyper::Error),
#[serde(skip)]
Http(#[from] http::Error),
#[serde(skip)]
Serde(#[from] serde_json::Error),
#[serde(skip)]
Fmt(#[from] std::fmt::Error),
#[serde(skip)]
Url(#[from] url::ParseError),
}
trait IntoBoxedError: std::error::Error {}
impl IntoBoxedError for net::AddrParseError {}
impl IntoBoxedError for tokio::task::JoinError {}
impl IntoBoxedError for api4::databuffer_tools::FindActiveError {}
impl<E> From<E> for RetrievalError
where
E: ToString + IntoBoxedError,
{
fn from(value: E) -> Self {
Self::TextError(value.to_string())
}
}
pub async fn host(node_config: NodeConfigCached) -> Result<(), RetrievalError> {
static STATUS_BOARD_INIT: Once = Once::new();
STATUS_BOARD_INIT.call_once(|| {
let b = StatusBoard::new();
@@ -84,7 +114,7 @@ pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
let node_config = node_config.clone();
let addr = conn.remote_addr();
async move {
Ok::<_, Error>(service_fn({
Ok::<_, RetrievalError>(service_fn({
move |req| {
// TODO send to logstash
info!(
@@ -106,7 +136,7 @@ pub async fn host(node_config: NodeConfigCached) -> Result<(), Error> {
Ok(())
}
async fn http_service(req: Request<Body>, node_config: NodeConfigCached) -> Result<Response<Body>, Error> {
async fn http_service(req: Request<Body>, node_config: NodeConfigCached) -> Result<Response<Body>, RetrievalError> {
match http_service_try(req, &node_config).await {
Ok(k) => Ok(k),
Err(e) => {
@@ -122,7 +152,7 @@ struct Cont<F> {
impl<F, I> Future for Cont<F>
where
F: Future<Output = Result<I, Error>>,
F: Future<Output = Result<I, RetrievalError>>,
{
type Output = <F as Future>::Output;
@@ -131,14 +161,14 @@ where
match h {
Ok(k) => k,
Err(e) => {
error!("Cont<F> catch_unwind {:?}", e);
match e.downcast_ref::<Error>() {
error!("Cont<F> catch_unwind {e:?}");
match e.downcast_ref::<RetrievalError>() {
Some(e) => {
error!("Cont<F> catch_unwind is Error: {:?}", e);
error!("Cont<F> catch_unwind is Error: {e:?}");
}
None => {}
}
Poll::Ready(Err(Error::with_msg(format!("{:?}", e))))
Poll::Ready(Err(RetrievalError::TextError(format!("{e:?}"))))
}
}
}
@@ -182,7 +212,7 @@ impl ReqCtx {
}
// TODO remove because I want error bodies to be json.
pub fn response_err<T>(status: StatusCode, msg: T) -> Result<Response<Body>, Error>
pub fn response_err<T>(status: StatusCode, msg: T) -> Result<Response<Body>, RetrievalError>
where
T: AsRef<str>,
{
@@ -241,7 +271,10 @@ macro_rules! static_http_api1 {
};
}
async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
async fn http_service_try(
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
use http::HeaderValue;
let mut urlmarks = Vec::new();
urlmarks.push(format!("{}:{}", req.method(), req.uri()));
@@ -271,7 +304,7 @@ async fn http_service_inner(
req: Request<Body>,
ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let uri = req.uri().clone();
let path = uri.path();
if path == "/api/4/private/version" {
@@ -313,6 +346,8 @@ async fn http_service_inner(
h.handle(req, ctx, &node_config).await
} else if let Some(h) = StatusBoardAllHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = api4::databuffer_tools::FindActiveHandler::handler(&req) {
Ok(h.handle(req, &node_config).await?)
} else if let Some(h) = api4::search::ChannelSearchHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = api4::binned::BinnedHandler::handler(&req) {
@@ -436,7 +471,7 @@ async fn http_service_inner(
}
}
pub fn api_4_docs(path: &str) -> Result<Response<Body>, Error> {
pub fn api_4_docs(path: &str) -> Result<Response<Body>, RetrievalError> {
static_http!(path, "", "api4.html", "text/html");
static_http!(path, "style.css", "text/css");
static_http!(path, "script.js", "text/javascript");
@@ -444,7 +479,7 @@ pub fn api_4_docs(path: &str) -> Result<Response<Body>, Error> {
Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
}
pub fn api_1_docs(path: &str) -> Result<Response<Body>, Error> {
pub fn api_1_docs(path: &str) -> Result<Response<Body>, RetrievalError> {
static_http_api1!(path, "", "api1.html", "text/html");
static_http_api1!(path, "style.css", "text/css");
static_http_api1!(path, "script.js", "text/javascript");
@@ -462,7 +497,11 @@ impl StatusBoardAllHandler {
}
}
pub async fn handle(&self, _req: Request<Body>, _node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
pub async fn handle(
&self,
_req: Request<Body>,
_node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
use std::ops::Deref;
let sb = status_board().unwrap();
let buf = serde_json::to_vec(sb.deref()).unwrap();
@@ -471,12 +510,16 @@ impl StatusBoardAllHandler {
}
}
async fn prebinned(req: Request<Body>, ctx: &ReqCtx, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
async fn prebinned(
req: Request<Body>,
ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, RetrievalError> {
match prebinned_inner(req, ctx, node_config).await {
Ok(ret) => Ok(ret),
Err(e) => {
error!("fn prebinned: {e:?}");
Ok(response(StatusCode::BAD_REQUEST).body(Body::from(e.msg().to_string()))?)
Ok(response(StatusCode::BAD_REQUEST).body(Body::from(format!("[prebinned-error]")))?)
}
}
}
@@ -485,7 +528,7 @@ async fn prebinned_inner(
req: Request<Body>,
_ctx: &ReqCtx,
_node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let url: url::Url = format!("dummy://{}", head.uri).parse()?;
let query = PreBinnedQuery::from_url(&url)?;
@@ -493,7 +536,7 @@ async fn prebinned_inner(
span1.in_scope(|| {
debug!("begin");
});
error!("TODO hhtpret prebinned_inner");
error!("TODO httpret prebinned_inner");
//let fut = disk::binned::prebinned::pre_binned_bytes_for_http(node_config, &query).instrument(span1);
todo!()
}
@@ -502,7 +545,7 @@ async fn random_channel(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (_head, _body) = req.into_parts();
let ret = dbconn::random_channel(node_config).await?;
let ret = response(StatusCode::OK).body(Body::from(ret))?;
@@ -513,7 +556,7 @@ async fn clear_cache_all(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let dry = match head.uri.query() {
Some(q) => q.contains("dry"),
@@ -530,7 +573,7 @@ async fn update_db_with_channel_names(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
info!("httpret::update_db_with_channel_names");
let (head, _body) = req.into_parts();
let _dry = match head.uri.query() {
@@ -568,7 +611,7 @@ async fn update_db_with_channel_names_3(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let _dry = match head.uri.query() {
Some(q) => q.contains("dry"),
@@ -591,7 +634,7 @@ async fn update_db_with_all_channel_configs(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let _dry = match head.uri.query() {
Some(q) => q.contains("dry"),
@@ -614,7 +657,7 @@ async fn update_search_cache(
req: Request<Body>,
_ctx: &ReqCtx,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
) -> Result<Response<Body>, RetrievalError> {
let (head, _body) = req.into_parts();
let _dry = match head.uri.query() {
Some(q) => q.contains("dry"),
@@ -638,8 +681,9 @@ pub struct StatusBoardEntry {
is_error: bool,
#[serde(skip_serializing_if = "is_false")]
is_ok: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
errors: Vec<Error>,
// #[serde(skip_serializing_if = "Vec::is_empty")]
#[serde(skip)]
errors: Vec<Box<dyn std::error::Error + Send>>,
}
mod instant_serde {
@@ -693,10 +737,10 @@ impl StatusBoard {
use std::io::Read;
self.clean();
let mut f = File::open("/dev/urandom").unwrap();
let mut buf = [0; 8];
let mut buf = [0; 4];
f.read_exact(&mut buf).unwrap();
let n = u64::from_le_bytes(buf);
let s = format!("{:016x}", n);
let n = u32::from_le_bytes(buf);
let s = format!("{:08x}", n);
info!("new_status_id {s}");
self.entries.insert(s.clone(), StatusBoardEntry::new());
s
@@ -738,13 +782,16 @@ impl StatusBoard {
}
}
pub fn add_error(&mut self, status_id: &str, error: Error) {
pub fn add_error<E>(&mut self, status_id: &str, error: E)
where
E: Into<Box<dyn std::error::Error + Send>>,
{
match self.entries.get_mut(status_id) {
Some(e) => {
e.ts_updated = SystemTime::now();
e.is_error = true;
e.is_ok = false;
e.errors.push(error);
e.errors.push(error.into());
}
None => {
error!("can not find status id {}", status_id);
@@ -764,7 +811,9 @@ impl StatusBoard {
let js = StatJs { errors: Vec::new() };
return serde_json::to_string(&js).unwrap();
} else if e.is_error {
let errors = e.errors.iter().map(|e| (&e.0).into()).collect();
// TODO
// let errors = e.errors.iter().map(|e| (&e.0).into()).collect();
let errors = vec![err::Error::with_msg_no_trace("TODO convert to user error").into()];
let js = StatJs { errors };
return serde_json::to_string(&js).unwrap();
} else {
@@ -785,10 +834,10 @@ impl StatusBoard {
static STATUS_BOARD: AtomicPtr<RwLock<StatusBoard>> = AtomicPtr::new(std::ptr::null_mut());
pub fn status_board() -> Result<RwLockWriteGuard<'static, StatusBoard>, Error> {
pub fn status_board() -> Result<RwLockWriteGuard<'static, StatusBoard>, RetrievalError> {
let x = unsafe { &*STATUS_BOARD.load(Ordering::SeqCst) }.write();
match x {
Ok(x) => Ok(x),
Err(e) => Err(Error::with_msg(format!("{e:?}"))),
Err(e) => Err(RetrievalError::TextError(format!("{e}"))),
}
}
crates/httpret/src/prometheus.rs
+26 -17
View File
@@ -1,11 +1,20 @@
use crate::err::Error;
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode};
use crate::RetrievalError;
use http::HeaderMap;
use http::HeaderValue;
use http::Method;
use http::Request;
use http::Response;
use http::StatusCode;
use hyper::server::conn::AddrStream;
use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Server};
use hyper::service::make_service_fn;
use hyper::service::service_fn;
use hyper::Body;
use hyper::Server;
use netpod::log::*;
use netpod::{ACCEPT_ALL, APP_JSON};
use serde_json::{json, Value};
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
use serde_json::json;
use serde_json::Value;
use std::collections::BTreeMap;
use std::net::SocketAddr;
use std::sync::Once;
@@ -50,7 +59,7 @@ impl StatusBuildInfoHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
info!("{} for {:?}", std::any::type_name::<Self>(), req);
if req.method() == Method::GET {
if accepts_json(req.headers()) {
@@ -91,7 +100,7 @@ impl SeriesHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
info!("{} for {:?}", std::any::type_name::<Self>(), req);
if req.method() == Method::GET || req.method() == Method::POST {
if accepts_json(req.headers()) {
@@ -128,7 +137,7 @@ impl MetadataHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
info!("{} for {:?}", std::any::type_name::<Self>(), req);
if req.method() == Method::GET {
if accepts_json(req.headers()) {
@@ -163,7 +172,7 @@ impl LabelsHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
let self_name = std::any::type_name::<Self>();
info!("{} for {:?}", self_name, req);
if req.method() == Method::GET || req.method() == Method::POST {
@@ -218,7 +227,7 @@ impl LabelValuesHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
let self_name = std::any::type_name::<Self>();
info!("{} for {:?}", self_name, req);
info!("LABEL {:?}", self.label);
@@ -263,7 +272,7 @@ impl QueryHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
info!("{} for {:?}", std::any::type_name::<Self>(), req);
let url = url::Url::parse(&format!("dummy://{}", &req.uri()));
info!("/api/v1/query parsed url: {:?}", url);
@@ -295,7 +304,7 @@ impl QueryRangeHandler {
}
}
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
pub async fn handle(&self, req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
info!("{} for {:?}", std::any::type_name::<Self>(), req);
let url = url::Url::parse(&format!("dummy://{}", &req.uri()));
info!("/api/v1/query_range parsed url: {:?}", url);
@@ -375,7 +384,7 @@ impl QueryRangeHandler {
}
}
async fn http_service_inner(req: Request<Body>) -> Result<Response<Body>, Error> {
async fn http_service_inner(req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
if let Some(h) = StatusBuildInfoHandler::handler(&req) {
h.handle(req).await
} else if let Some(h) = SeriesHandler::handler(&req) {
@@ -396,7 +405,7 @@ async fn http_service_inner(req: Request<Body>) -> Result<Response<Body>, Error>
}
}
async fn http_service(req: Request<Body>) -> Result<Response<Body>, Error> {
async fn http_service(req: Request<Body>) -> Result<Response<Body>, RetrievalError> {
match http_service_inner(req).await {
Ok(k) => Ok(k),
Err(e) => {
@@ -406,12 +415,12 @@ async fn http_service(req: Request<Body>) -> Result<Response<Body>, Error> {
}
}
pub async fn host(bind: SocketAddr) -> Result<(), Error> {
pub async fn host(bind: SocketAddr) -> Result<(), RetrievalError> {
let make_service = make_service_fn({
move |conn: &AddrStream| {
let addr = conn.remote_addr();
async move {
Ok::<_, Error>(service_fn({
Ok::<_, RetrievalError>(service_fn({
move |req| {
info!(
"REQUEST {:?} - {:?} - {:?} - {:?}",
crates/httpret/src/proxy.rs
+1 -1
View File
@@ -6,7 +6,6 @@ use crate::api1::channel_search_list_v1;
use crate::api1::gather_json_2_v1;
use crate::api_1_docs;
use crate::api_4_docs;
use crate::err::Error;
use crate::gather::gather_get_json_generic;
use crate::gather::SubRes;
use crate::pulsemap::MapPulseQuery;
@@ -15,6 +14,7 @@ use crate::response_err;
use crate::Cont;
use crate::ReqCtx;
use crate::PSI_DAQBUFFER_SERVICE_MARK;
use err::Error;
use futures_util::pin_mut;
use futures_util::Stream;
use http::Method;
crates/httpret/src/proxy/api1.rs
+4 -5
View File
@@ -1,8 +1,8 @@
pub mod reqstatus;
use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use err::Error;
use http::HeaderValue;
use http::Method;
use http::Request;
@@ -14,6 +14,7 @@ use netpod::log::*;
use netpod::query::api1::Api1Query;
use netpod::ProxyConfig;
use netpod::ACCEPT_ALL;
use netpod::X_DAQBUF_REQID;
pub struct PythonDataApi1Query {}
@@ -85,10 +86,8 @@ impl PythonDataApi1Query {
} else {
info!("backend returned OK");
let riq_def = HeaderValue::from_static("(none)");
let riq = head.headers.get("x-daqbuffer-request-id").unwrap_or(&riq_def);
Ok(response(StatusCode::OK)
.header("x-daqbuffer-request-id", riq)
.body(body)?)
let riq = head.headers.get(X_DAQBUF_REQID).unwrap_or(&riq_def);
Ok(response(StatusCode::OK).header(X_DAQBUF_REQID, riq).body(body)?)
}
} else {
Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(Body::empty())?)
crates/httpret/src/proxy/api1/reqstatus.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use crate::bodystream::response;
use crate::err::Error;
use err::Error;
use http::Method;
use http::Request;
use http::Response;
crates/httpret/src/proxy/api4.rs
+1 -1
View File
@@ -1,12 +1,12 @@
pub mod caioclookup;
use crate::bodystream::ToPublicResponse;
use crate::err::Error;
use crate::gather::gather_get_json_generic;
use crate::gather::SubRes;
use crate::gather::Tag;
use crate::response;
use crate::ReqCtx;
use err::Error;
use futures_util::Future;
use http::Method;
use http::Request;
crates/httpret/src/proxy/api4/caioclookup.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::bodystream::response;
use crate::err::Error;
use crate::ReqCtx;
use err::Error;
use http::Request;
use http::Response;
use http::StatusCode;
crates/httpret/src/pulsemap.rs
+1 -1
View File
@@ -1,4 +1,3 @@
use crate::err::Error;
use crate::response;
use async_channel::Receiver;
use async_channel::Sender;
@@ -7,6 +6,7 @@ use bytes::BufMut;
use bytes::BytesMut;
use chrono::TimeZone;
use chrono::Utc;
use err::Error;
use futures_util::stream::FuturesOrdered;
use futures_util::stream::FuturesUnordered;
use futures_util::FutureExt;
crates/httpret/src/settings.rs
+8 -4
View File
@@ -1,10 +1,14 @@
use crate::err::Error;
use crate::response;
use http::{Method, StatusCode};
use hyper::{Body, Request, Response};
use err::Error;
use http::Method;
use http::StatusCode;
use hyper::Body;
use hyper::Request;
use hyper::Response;
use netpod::log::*;
use netpod::NodeConfigCached;
use netpod::{ACCEPT_ALL, APP_JSON};
use netpod::ACCEPT_ALL;
use netpod::APP_JSON;
pub struct SettingsThreadsMaxHandler {}