daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes and timeout handler

Browse Source
This commit is contained in:
Dominik Werder
2022-09-08 18:19:53 +02:00
parent eff80450ee
commit 62a1be0c4b
10 changed files with 275 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
httpret/src/channel_status.rs
View File

@@ -7,11 +7,65 @@ use netpod::query::ChannelStateEventsQuery;
use netpod::{FromUrl, NodeConfigCached, ACCEPT_ALL, APP_JSON};
use url::Url;
pub struct ChannelStatusConnectionEvents {}
pub struct ConnectionStatusEvents {}
impl ChannelStatusConnectionEvents {
impl ConnectionStatusEvents {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/channel/status/connection/events" {
if req.uri().path() == "/api/4/scylla/connection/status/events" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() == Method::GET {
let accept_def = APP_JSON;
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept == APP_JSON || accept == ACCEPT_ALL {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelStateEventsQuery::from_url(&url)?;
match self.fetch_data(&q, node_config).await {
Ok(k) => {
let body = Body::from(serde_json::to_vec(&k)?);
Ok(response(StatusCode::OK).body(body)?)
}
Err(e) => Ok(response(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{:?}", e.public_msg())))?),
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
}
async fn fetch_data(
&self,
q: &ChannelStateEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Vec<(u64, u32)>, Error> {
let dbconf = &node_config.node_config.cluster.database;
let scyco = node_config
.node_config
.cluster
.scylla
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("No Scylla configured")))?;
let ret = channel_state_events(q, scyco, dbconf.clone()).await?;
Ok(ret)
}
}
pub struct ChannelConnectionStatusEvents {}
impl ChannelConnectionStatusEvents {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/scylla/channel/connection/status/events" {
Some(Self {})
} else {
None

90
httpret/src/events.rs
View File

@@ -4,7 +4,7 @@ use crate::{response, response_err, BodyStream, ToPublicResponse};
use futures_util::{Stream, StreamExt, TryStreamExt};
use http::{Method, Request, Response, StatusCode};
use hyper::Body;
use items_2::{binned_collected, ChannelEvents, ChannelEventsMerger};
use items_2::{binned_collected, empty_events_dyn, empty_events_dyn_2, ChannelEvents, ChannelEventsMerger};
use netpod::log::*;
use netpod::query::{BinnedQuery, ChannelStateEventsQuery, PlainEventsQuery};
use netpod::{AggKind, BinnedRange, FromUrl, NodeConfigCached};
@@ -14,6 +14,7 @@ use scyllaconn::errconv::ErrConv;
use scyllaconn::events::{channel_state_events, find_series, make_scylla_stream};
use std::pin::Pin;
use std::sync::Arc;
use std::time::Instant;
use url::Url;
pub struct EventsHandler {}
@@ -164,11 +165,13 @@ impl EventsHandlerScylla {
}
async fn gather(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let self_name = std::any::type_name::<Self>();
let (head, _body) = req.into_parts();
warn!("TODO PlainEventsQuery needs to take AggKind");
warn!("TODO PlainEventsQuery needs to take AggKind to do x-binning");
let s1 = format!("dummy:{}", head.uri);
let url = Url::parse(&s1)?;
let evq = PlainEventsQuery::from_url(&url)?;
let deadline = Instant::now() + evq.timeout();
let pgclient = {
// TODO use common connection/pool:
info!("--------------- open postgres connection");
@@ -182,16 +185,50 @@ impl EventsHandlerScylla {
let pgclient = Arc::new(pgclient);
pgclient
};
let mut stream = if let Some(scyco) = &node_config.node_config.cluster.scylla {
let scy = create_scy_session(scyco).await?;
let (series, scalar_type, shape) = { find_series(evq.channel(), pgclient.clone()).await? };
let stream = make_scylla_stream(&evq, series, scalar_type.clone(), shape.clone(), scy, false).await?;
stream
let scyco = if let Some(scyco) = &node_config.node_config.cluster.scylla {
scyco
} else {
return Err(Error::with_public_msg(format!("no scylla configured")));
};
let scy = create_scy_session(scyco).await?;
let do_one_before_range = evq.do_one_before_range();
let (series, scalar_type, shape) = find_series(evq.channel(), pgclient.clone()).await?;
let empty_item = empty_events_dyn_2(&scalar_type, &shape, &AggKind::TimeWeightedScalar);
let empty_stream =
futures_util::stream::once(futures_util::future::ready(Ok(ChannelEvents::Events(empty_item))));
let stream2 = make_scylla_stream(
&evq,
do_one_before_range,
series,
scalar_type.clone(),
shape.clone(),
scy,
false,
)
.await?;
let mut stream = empty_stream.chain(stream2);
let mut coll = None;
while let Some(item) = stream.next().await {
let mut fut = None;
loop {
// Alternative way, test what works better:
if fut.is_none() {
fut = Some(stream.next());
}
let item = match tokio::time::timeout_at(deadline.into(), fut.as_mut().unwrap()).await {
Ok(Some(item)) => {
fut.take();
item
}
Ok(None) => {
fut.take();
break;
}
Err(_) => {
warn!("{self_name} timeout");
fut.take();
break;
}
};
match item {
Ok(k) => match k {
ChannelEvents::Events(mut item) => {
@@ -218,10 +255,13 @@ impl EventsHandlerScylla {
Ok(ret)
}
None => {
let ret = response(StatusCode::OK).body(BodyStream::wrapped(
futures_util::stream::iter([Ok(Vec::new())]),
format!("EventsHandlerScylla::gather"),
))?;
error!("should never happen with changed logic, remove case");
err::todo();
let item = empty_events_dyn(&scalar_type, &shape, &AggKind::TimeWeightedScalar);
let res = item.to_box_to_json_result();
let res = res.to_json_result()?;
let res = res.to_json_bytes()?;
let ret = response(StatusCode::OK).body(Body::from(res))?;
Ok(ret)
}
}
@@ -269,10 +309,11 @@ impl BinnedHandlerScylla {
async fn gather(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
warn!("TODO BinnedQuery needs to take AggKind");
warn!("TODO BinnedQuery needs to take AggKind to do x-binngin");
let s1 = format!("dummy:{}", head.uri);
let url = Url::parse(&s1)?;
let evq = BinnedQuery::from_url(&url)?;
let do_one_before_range = evq.agg_kind().need_expand();
let pgclient = {
// TODO use common connection/pool:
info!("--------------- open postgres connection");
@@ -288,13 +329,23 @@ impl BinnedHandlerScylla {
};
if let Some(scyco) = &node_config.node_config.cluster.scylla {
let scy = create_scy_session(scyco).await?;
let mut query2 = PlainEventsQuery::new(evq.channel().clone(), evq.range().clone(), 0, None, false);
let covering = BinnedRange::covering_range(evq.range().clone(), evq.bin_count())?;
let range = covering.full_range();
let mut query2 = PlainEventsQuery::new(evq.channel().clone(), range.clone(), 0, None, false);
query2.set_timeout(evq.timeout());
let query2 = query2;
let (series, scalar_type, shape) = { find_series(evq.channel(), pgclient.clone()).await? };
let stream =
make_scylla_stream(&query2, series, scalar_type.clone(), shape.clone(), scy.clone(), false).await?;
let query3 = ChannelStateEventsQuery::new(evq.channel().clone(), evq.range().clone());
let (series, scalar_type, shape) = find_series(evq.channel(), pgclient.clone()).await?;
let stream = make_scylla_stream(
&query2,
do_one_before_range,
series,
scalar_type.clone(),
shape.clone(),
scy.clone(),
false,
)
.await?;
let query3 = ChannelStateEventsQuery::new(evq.channel().clone(), range.clone());
let state_stream = channel_state_events(&query3, scy.clone())
.await?
.map(|x| {
@@ -313,9 +364,6 @@ impl BinnedHandlerScylla {
let state_stream = Box::pin(state_stream) as _;
let merged_stream = ChannelEventsMerger::new(vec![data_stream, state_stream]);
let merged_stream = Box::pin(merged_stream) as Pin<Box<dyn Stream<Item = _> + Send>>;
let covering = BinnedRange::covering_range(evq.range().clone(), evq.bin_count())?;
//eprintln!("edges {:?}", covering.edges());
// TODO return partial result if timed out.
let binned_collected = binned_collected(
scalar_type.clone(),
shape.clone(),

5
httpret/src/httpret.rs
View File

@@ -18,7 +18,6 @@ use crate::bodystream::response;
use crate::err::Error;
use crate::gather::gather_get_json;
use crate::pulsemap::UpdateTask;
use channel_status::ChannelStatusConnectionEvents;
use channelconfig::{chconf_from_binned, ChConf};
use disk::binned::query::PreBinnedQuery;
use future::Future;
@@ -249,7 +248,9 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = events::BinnedHandlerScylla::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = ChannelStatusConnectionEvents::handler(&req) {
} else if let Some(h) = channel_status::ConnectionStatusEvents::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channel_status::ChannelConnectionStatusEvents::handler(&req) {
h.handle(req, &node_config).await
} else if path == "/api/4/binned" {
if req.method() == Method::GET {

156
httpret/static/documentation/api4.html
View File

@@ -239,12 +239,11 @@ curl -H 'Accept: application/json' 'https://data-api.psi.ch/api/4/events?channel
<li>channelName (e.g. "SLAAR-LSCP4-LAS6891:CH7:1")</li>
<li>begDate (e.g. "2021-05-26T07:10:00.000Z")</li>
<li>endDate (e.g. "2021-05-26T07:16:00.000Z")</li>
<li>binCount (number of requested bins in time-dimension, e.g. "6". The actual number of returned bins depends
on
bin-cache-grid-resolution. The server tries to find the best match.)</li>
<li>binCount (number of requested bins in time-dimension, e.g. "6". The actual number of returned bins depends on
bin-cache-grid-resolution. The server tries to find a reasonable match.)</li>
<li>binningScheme (optional)</li>
<ul>
<li>if not specified: default is "binningScheme=unweightedScalar".</li>
<li>if not specified: default is "binningScheme=timeWeightedScalar".</li>
<li>"binningScheme=unweightedScalar": non-weighted binning, waveform gets first averaged to a scalar.</li>
<li>"binningScheme=timeWeightedScalar": time-weighted binning, waveform gets first averaged to a scalar.
</li>
@@ -272,159 +271,38 @@ curl -H 'Accept: application/json' 'https://data-api.psi.ch/api/4/binned?channel
<h4>Example response (without usage of binningScheme):</h4>
<pre>{
"avgs": [
16204.087890625,
16204.3798828125,
16203.9296875,
16204.232421875,
16202.974609375,
16203.208984375,
16203.4345703125
],
"counts": [
1000,
999,
1000,
999,
1000,
999,
1000
],
"finalisedRange": true,
"maxs": [
48096,
48100,
48094,
48096,
48096,
48095,
48096
],
"mins": [
0,
0,
0,
0,
0,
0,
0
],
"tsAnchor": 1623769850,
"tsMs": [
0,
10000,
20000,
30000,
40000,
50000,
60000,
70000
20000
],
"tsNs": [
0,
0,
0,
0,
0,
0,
0,
0
]
}
</pre>
<h4>Example response (waveform channel and usage of binningScheme):</h4>
<pre>{
"tsAnchor": 1623769950,
"tsMs": [
0,
10000,
20000,
30000,
40000,
50000,
60000,
70000
],
"tsNs": [
0,
0,
0,
0,
0,
0,
0,
0
],
"finalisedRange": true,
"counts": [
1000,
1000,
...
999,
1000
],
"avgs": [
[
0.013631398789584637,
34936.76953125,
45045.5078125,
31676.30859375,
880.7999877929688,
576.4010620117188,
295.1236877441406
],
[
0.01851877197623253,
34935.734375,
45044.2734375,
31675.359375,
880.7310791015625,
576.3038330078125,
295.06134033203125
],
...
16204.087890625,
16204.3798828125,
16203.9296875
],
"maxs": [
[
111,
48093,
45804,
47122,
1446,
783,
431
],
[
120,
48092,
45803,
47124,
1452,
782,
431
],
...
48096,
48100,
48094
],
"mins": [
[
0,
0,
44329,
267,
519,
394,
0
],
[
0,
0,
44327,
265,
514,
395,
0
],
...
]
0,
0,
0
],
"finalisedRange": true
}
</pre>