daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2023-06-27 17:02:37 +02:00
parent f4c1539c37
commit 8938e55f86
19 changed files with 597 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
httpret/src/api1.rs
View File

@@ -51,6 +51,7 @@ use query::transform::TransformQuery;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
use std::any;
use std::collections::VecDeque;
use std::future::Future;
use std::pin::Pin;
@@ -518,7 +519,7 @@ async fn find_ch_conf(
range: NanoRange,
channel: SfDbChannel,
ncc: NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret = nodenet::channelconfig::channel_config(range, channel, &ncc).await?;
Ok(ret)
}
@@ -530,7 +531,7 @@ pub struct DataApiPython3DataStream {
current_channel: Option<ChannelTypeConfigGen>,
node_config: NodeConfigCached,
chan_stream: Option<Pin<Box<dyn Stream<Item = Result<BytesMut, Error>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<ChannelTypeConfigGen, Error>> + Send>>>,
config_fut: Option<Pin<Box<dyn Future<Output = Result<Option<ChannelTypeConfigGen>, Error>> + Send>>>,
disk_io_tune: DiskIoTune,
do_decompress: bool,
#[allow(unused)]
@@ -676,7 +677,6 @@ impl DataApiPython3DataStream {
// TODO this stream can currently only handle sf-databuffer type backend anyway.
fn handle_config_fut_ready(&mut self, fetch_info: SfChFetchInfo) -> Result<(), Error> {
self.config_fut = None;
debug!("found channel_config {:?}", fetch_info);
let select = EventsSubQuerySelect::new(
ChannelTypeConfigGen::SfDatabuffer(fetch_info.clone()),
self.range.clone().into(),
@@ -695,7 +695,7 @@ impl DataApiPython3DataStream {
let event_chunker_conf = EventChunkerConf::new(ByteSize::from_kb(1024));
let s = make_local_event_blobs_stream(
self.range.clone(),
&fetch_info,
fetch_info.clone(),
one_before,
self.do_decompress,
event_chunker_conf,
@@ -704,7 +704,7 @@ impl DataApiPython3DataStream {
)?;
Box::pin(s) as Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>
} else {
debug!("Set up merged remote stream");
debug!("set up merged remote stream {}", fetch_info.name());
let s = MergedBlobsFromRemotes::new(subq, self.node_config.node_config.cluster.clone());
Box::pin(s) as Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>
};
@@ -762,7 +762,7 @@ impl Stream for DataApiPython3DataStream {
}
} else if let Some(fut) = &mut self.config_fut {
match fut.poll_unpin(cx) {
Ready(Ok(k)) => match k {
Ready(Ok(Some(k))) => match k {
ChannelTypeConfigGen::Scylla(_) => {
let e = Error::with_msg_no_trace("scylla");
error!("{e}");
@@ -779,6 +779,11 @@ impl Stream for DataApiPython3DataStream {
}
},
},
Ready(Ok(None)) => {
warn!("logic error");
self.config_fut = None;
continue;
}
Ready(Err(e)) => {
self.data_done = true;
Ready(Some(Err(e)))
@@ -795,7 +800,7 @@ impl Stream for DataApiPython3DataStream {
self.node_config.clone(),
)));
}
self.config_fut = Some(Box::pin(futures_util::future::ready(Ok(channel))));
self.config_fut = Some(Box::pin(futures_util::future::ready(Ok(Some(channel)))));
continue;
} else {
self.data_done = true;
@@ -858,7 +863,7 @@ impl Api1EventsBinaryHandler {
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?
.to_owned();
let body_data = hyper::body::to_bytes(body).await?;
if body_data.len() < 512 && body_data.first() == Some(&"{".as_bytes()[0]) {
if body_data.len() < 1024 * 2 && body_data.first() == Some(&"{".as_bytes()[0]) {
info!("request body_data string: {}", String::from_utf8_lossy(&body_data));
}
let qu = match serde_json::from_slice::<Api1Query>(&body_data) {
@@ -895,9 +900,10 @@ impl Api1EventsBinaryHandler {
span: tracing::Span,
ncc: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let self_name = any::type_name::<Self>();
// TODO this should go to usage statistics:
info!(
"Handle Api1Query {:?} {} {:?}",
"{self_name} {:?} {} {:?}",
qu.range(),
qu.channels().len(),
qu.channels().first()
@@ -905,7 +911,7 @@ impl Api1EventsBinaryHandler {
let settings = EventsSubQuerySettings::from(&qu);
let beg_date = qu.range().beg().clone();
let end_date = qu.range().end().clone();
trace!("Api1Query beg_date {:?} end_date {:?}", beg_date, end_date);
trace!("{self_name} beg_date {:?} end_date {:?}", beg_date, end_date);
//let url = Url::parse(&format!("dummy:{}", req.uri()))?;
//let query = PlainEventsBinaryQuery::from_url(&url)?;
if accept.contains(APP_OCTET) || accept.contains(ACCEPT_ALL) {
@@ -916,12 +922,25 @@ impl Api1EventsBinaryHandler {
let backend = &ncc.node_config.cluster.backend;
// TODO ask for channel config quorum for all channels up front.
//httpclient::http_get(url, accept);
let ts1 = Instant::now();
let mut chans = Vec::new();
for ch in qu.channels() {
info!("try to find config quorum for {ch:?}");
let ch = SfDbChannel::from_name(backend, ch.name());
let ch_conf = nodenet::configquorum::find_config_basics_quorum(ch, range.clone().into(), ncc).await?;
chans.push(ch_conf);
let ch_conf =
nodenet::configquorum::find_config_basics_quorum(ch.clone(), range.clone().into(), ncc).await?;
match ch_conf {
Some(x) => {
chans.push(x);
}
None => {
error!("no config quorum found for {ch:?}");
}
}
}
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1).as_millis();
info!("{self_name} configs fetched in {} ms", dt);
// TODO use a better stream protocol with built-in error delivery.
let status_id = super::status_board()?.new_status_id();
let s = DataApiPython3DataStream::new(
@@ -942,8 +961,8 @@ impl Api1EventsBinaryHandler {
Ok(ret)
} else {
// TODO set the public error code and message and return Err(e).
let e = Error::with_public_msg(format!("Unsupported Accept: {}", accept));
error!("{e}");
let e = Error::with_public_msg(format!("{self_name} unsupported Accept: {}", accept));
error!("{self_name} {e}");
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?)
}
}

5
httpret/src/api4/binned.rs
View File

@@ -27,7 +27,10 @@ async fn binned_json(url: Url, req: Request<Body>, node_config: &NodeConfigCache
let msg = format!("can not parse query: {}", e.msg());
e.add_public_msg(msg)
})?;
let ch_conf = ch_conf_from_binned(&query, node_config).await?;
// TODO handle None case better and return 404
let ch_conf = ch_conf_from_binned(&query, node_config)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
let span1 = span!(
Level::INFO,
"httpret::binned",

6
httpret/src/api4/events.rs
View File

@@ -93,7 +93,11 @@ async fn plain_events_json(
let (_head, _body) = req.into_parts();
let query = PlainEventsQuery::from_url(&url)?;
info!("plain_events_json query {query:?}");
let ch_conf = chconf_from_events_v1(&query, node_config).await.map_err(Error::from)?;
// TODO handle None case better and return 404
let ch_conf = chconf_from_events_v1(&query, node_config)
.await
.map_err(Error::from)?
.ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
info!("plain_events_json chconf_from_events_v1: {ch_conf:?}");
let item = streams::plaineventsjson::plain_events_json(&query, ch_conf, &node_config.node_config.cluster).await;
let item = match item {

5
httpret/src/channel_status.rs
View File

@@ -149,8 +149,9 @@ impl ChannelStatusEvents {
.as_ref()
.ok_or_else(|| Error::with_public_msg_no_trace(format!("no scylla configured")))?;
let scy = scyllaconn::create_scy_session(scyco).await?;
let chconf =
nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config).await?;
let chconf = nodenet::channelconfig::channel_config(q.range().clone(), q.channel().clone(), node_config)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel config not found"))?;
let do_one_before_range = true;
match chconf {
ChannelTypeConfigGen::Scylla(ch_conf) => {

83
httpret/src/channelconfig.rs
View File

@@ -36,18 +36,24 @@ use url::Url;
pub async fn chconf_from_events_v1(
q: &PlainEventsQuery,
ncc: &NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
Ok(ret)
}
pub async fn chconf_from_prebinned(q: &PreBinnedQuery, ncc: &NodeConfigCached) -> Result<ChannelTypeConfigGen, Error> {
pub async fn chconf_from_prebinned(
q: &PreBinnedQuery,
ncc: &NodeConfigCached,
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret =
nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.patch().patch_range(), ncc).await?;
Ok(ret)
}
pub async fn ch_conf_from_binned(q: &BinnedQuery, ncc: &NodeConfigCached) -> Result<ChannelTypeConfigGen, Error> {
pub async fn ch_conf_from_binned(
q: &BinnedQuery,
ncc: &NodeConfigCached,
) -> Result<Option<ChannelTypeConfigGen>, Error> {
let ret = nodenet::configquorum::find_config_basics_quorum(q.channel().clone(), q.range().clone(), ncc).await?;
Ok(ret)
}
@@ -91,16 +97,24 @@ impl ChannelConfigHandler {
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
info!("channel_config");
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelConfigQuery::from_url(&url)?;
info!("channel_config for q {q:?}");
let conf = nodenet::channelconfig::channel_config(q.range.clone(), q.channel.clone(), node_config).await?;
let res: ChannelConfigResponse = conf.into();
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&res)?))?;
Ok(ret)
match conf {
Some(conf) => {
let res: ChannelConfigResponse = conf.into();
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&res)?))?;
Ok(ret)
}
None => {
let ret = response(StatusCode::NOT_FOUND)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::empty())?;
Ok(ret)
}
}
}
}
@@ -126,7 +140,7 @@ impl ChannelConfigsHandler {
match self.channel_configs(req, &node_config).await {
Ok(k) => Ok(k),
Err(e) => {
warn!("ChannelConfigHandler::handle: got error from channel_config: {e:?}");
warn!("got error from channel_config: {e}");
Ok(e.to_public_response())
}
}
@@ -151,6 +165,53 @@ impl ChannelConfigsHandler {
}
}
pub struct ChannelConfigQuorumHandler {}
impl ChannelConfigQuorumHandler {
pub fn handler(req: &Request<Body>) -> Option<Self> {
if req.uri().path() == "/api/4/channel/config/quorum" {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() == Method::GET {
let accept_def = APP_JSON;
let accept = req
.headers()
.get(http::header::ACCEPT)
.map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
match self.channel_config_quorum(req, &node_config).await {
Ok(k) => Ok(k),
Err(e) => {
warn!("from channel_config_quorum: {e}");
Ok(e.to_public_response())
}
}
} else {
Ok(response(StatusCode::BAD_REQUEST).body(Body::empty())?)
}
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
}
async fn channel_config_quorum(&self, req: Request<Body>, ncc: &NodeConfigCached) -> Result<Response<Body>, Error> {
info!("channel_config_quorum");
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let q = ChannelConfigQuery::from_url(&url)?;
info!("channel_config_quorum for q {q:?}");
let ch_confs = nodenet::configquorum::find_config_basics_quorum(q.channel, q.range.into(), ncc).await?;
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&ch_confs)?))?;
Ok(ret)
}
}
trait ErrConv<T> {
fn err_conv(self) -> Result<T, Error>;
}

2
httpret/src/httpret.rs
View File

@@ -317,6 +317,8 @@ async fn http_service_inner(
h.handle(req, &node_config).await
} else if let Some(h) = api4::binned::BinnedHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ChannelConfigQuorumHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ChannelConfigsHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelconfig::ChannelConfigHandler::handler(&req) {