daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor framed stream

Browse Source
This commit is contained in:
Dominik Werder
2024-07-24 12:06:30 +02:00
parent 06ac90aa70
commit a4228d03a4
27 changed files with 559 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
crates/nodenet/src/conn.rs
View File

@@ -1,7 +1,6 @@
use crate::scylla::scylla_channel_event_stream;
use bytes::Bytes;
use err::thiserror;
use err::Error;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
@@ -43,7 +42,16 @@ use tracing::Instrument;
mod test;
#[derive(Debug, ThisError)]
pub enum NodeNetError {}
#[cstm(name = "NodenetConn")]
pub enum Error {
BadQuery,
Scylla(#[from] crate::scylla::Error),
Error(#[from] err::Error),
Io(#[from] std::io::Error),
Items(#[from] items_2::Error),
NotAvailable,
DebugTest,
}
pub async fn events_service(ncc: NodeConfigCached) -> Result<(), Error> {
let scyqueue = err::todoval();
@@ -83,15 +91,17 @@ async fn make_channel_events_stream_data(
if subq.backend() == TEST_BACKEND {
let node_count = ncc.node_config.cluster.nodes.len() as u64;
let node_ix = ncc.ix as u64;
streams::generators::make_test_channel_events_stream_data(subq, node_count, node_ix)
let ret = streams::generators::make_test_channel_events_stream_data(subq, node_count, node_ix)?;
Ok(ret)
} else if let Some(scyqueue) = scyqueue {
let cfg = subq.ch_conf().to_scylla()?;
scylla_channel_event_stream(subq, cfg, scyqueue).await
let ret = scylla_channel_event_stream(subq, cfg, scyqueue).await?;
Ok(ret)
} else if let Some(_) = &ncc.node.channel_archiver {
let e = Error::with_msg_no_trace("archapp not built");
let e = Error::NotAvailable;
Err(e)
} else if let Some(_) = &ncc.node.archiver_appliance {
let e = Error::with_msg_no_trace("archapp not built");
let e = Error::NotAvailable;
Err(e)
} else {
let cfg = subq.ch_conf().to_sf_databuffer()?;
@@ -126,7 +136,7 @@ pub async fn create_response_bytes_stream(
debug!("wasm1 {:?}", evq.wasm1());
let reqctx = netpod::ReqCtx::new_from_single_reqid(evq.reqid().into()).into();
if evq.create_errors_contains("nodenet_parse_query") {
let e = Error::with_msg_no_trace("produced error on request nodenet_parse_query");
let e = Error::DebugTest;
return Err(e);
}
if evq.is_event_blobs() {
@@ -226,7 +236,7 @@ async fn events_conn_handler_with_reqid(
pub async fn events_get_input_frames<INP>(netin: INP) -> Result<Vec<InMemoryFrame>, Error>
where
INP: Stream<Item = Result<Bytes, Error>> + Unpin,
INP: Stream<Item = Result<Bytes, err::Error>> + Unpin,
{
let mut h = InMemoryFrameStream::new(netin, netpod::ByteSize::from_kb(8));
let mut frames = Vec::new();
@@ -243,7 +253,7 @@ where
debug!("ignored incoming frame {:?}", item);
}
Err(e) => {
return Err(e);
return Err(e.into());
}
}
}
@@ -254,12 +264,12 @@ pub fn events_parse_input_query(frames: Vec<InMemoryFrame>) -> Result<(EventsSub
if frames.len() != 1 {
error!("{:?}", frames);
error!("missing command frame len {}", frames.len());
let e = Error::with_msg("missing command frame");
let e = Error::BadQuery;
return Err(e);
}
let query_frame = &frames[0];
if query_frame.tyid() != EVENT_QUERY_JSON_STRING_FRAME {
return Err(Error::with_msg("query frame wrong type"));
return Err(Error::BadQuery);
}
// TODO this does not need all variants of Sitemty.
let qitem = match decode_frame::<Sitemty<EventQueryJsonStringFrame>>(query_frame) {
@@ -267,17 +277,17 @@ pub fn events_parse_input_query(frames: Vec<InMemoryFrame>) -> Result<(EventsSub
Ok(k) => match k {
StreamItem::DataItem(k) => match k {
RangeCompletableItem::Data(k) => k,
RangeCompletableItem::RangeComplete => return Err(Error::with_msg("bad query item")),
RangeCompletableItem::RangeComplete => return Err(Error::BadQuery),
},
_ => return Err(Error::with_msg("bad query item")),
_ => return Err(Error::BadQuery),
},
Err(e) => return Err(e),
Err(e) => return Err(e.into()),
},
Err(e) => return Err(e),
Err(e) => return Err(e.into()),
};
info!("parsing json {:?}", qitem.str());
let frame1: Frame1Parts = serde_json::from_str(&qitem.str()).map_err(|e| {
let e = Error::with_msg_no_trace(format!("json parse error: {} inp {}", e, qitem.str()));
let frame1: Frame1Parts = serde_json::from_str(&qitem.str()).map_err(|_e| {
let e = Error::BadQuery;
error!("{e}");
error!("input was {}", qitem.str());
e
@@ -293,7 +303,7 @@ async fn events_conn_handler_inner_try<INP>(
ncc: &NodeConfigCached,
) -> Result<(), ConnErr>
where
INP: Stream<Item = Result<Bytes, Error>> + Unpin,
INP: Stream<Item = Result<Bytes, err::Error>> + Unpin,
{
let _ = addr;
let frames = match events_get_input_frames(netin).await {
@@ -320,13 +330,13 @@ async fn events_conn_handler_inner<INP>(
ncc: &NodeConfigCached,
) -> Result<(), Error>
where
INP: Stream<Item = Result<Bytes, Error>> + Unpin,
INP: Stream<Item = Result<Bytes, err::Error>> + Unpin,
{
match events_conn_handler_inner_try(netin, netout, addr, scyqueue, ncc).await {
Ok(_) => (),
Err(ce) => {
let mut out = ce.netout;
let item: Sitemty<ChannelEvents> = Err(ce.err);
let item: Sitemty<ChannelEvents> = Err(err::Error::from_string(ce.err));
let buf = Framable::make_frame(&item)?;
out.write_all(&buf).await?;
}
@@ -350,7 +360,7 @@ async fn events_conn_handler(
Ok(k) => Ok(k),
Err(e) => {
error!("events_conn_handler sees error: {:?}", e);
Err(e)
Err(e.into())
}
}
}

23
crates/nodenet/src/scylla.rs
View File

@@ -1,4 +1,5 @@
use err::Error;
use err::thiserror;
use err::ThisError;
use futures_util::Stream;
use futures_util::StreamExt;
use futures_util::TryStreamExt;
@@ -9,16 +10,25 @@ use items_2::channelevents::ChannelEvents;
use netpod::log::*;
use netpod::ChConf;
use query::api4::events::EventsSubQuery;
use scyllaconn::events2::events::EventReadOpts;
use scyllaconn::events2::mergert;
use scyllaconn::worker::ScyllaQueue;
use scyllaconn::SeriesId;
use std::pin::Pin;
use taskrun::tokio;
#[derive(Debug, ThisError)]
#[cstm(name = "ScyllaChannelEventStream")]
pub enum Error {
MergeRt(#[from] mergert::Error),
}
pub async fn scylla_channel_event_stream(
evq: EventsSubQuery,
chconf: ChConf,
scyqueue: &ScyllaQueue,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
debug!("scylla_channel_event_stream {evq:?}");
// TODO depends in general on the query
// TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
// let do_one_before_range = evq.need_one_before_range();
@@ -27,7 +37,7 @@ pub async fn scylla_channel_event_stream(
let scalar_type = chconf.scalar_type();
let shape = chconf.shape();
let do_test_stream_error = false;
let with_values = evq.need_value_data();
let readopts = EventReadOpts::new(evq.need_value_data(), evq.transform().enum_as_string().unwrap_or(false));
let stream: Pin<Box<dyn Stream<Item = _> + Send>> = if let Some(rt) = evq.use_rt() {
let x = scyllaconn::events2::events::EventsStreamRt::new(
rt,
@@ -35,7 +45,7 @@ pub async fn scylla_channel_event_stream(
scalar_type.clone(),
shape.clone(),
evq.range().into(),
with_values,
readopts,
scyqueue.clone(),
)
.map_err(|e| scyllaconn::events2::mergert::Error::from(e));
@@ -46,7 +56,7 @@ pub async fn scylla_channel_event_stream(
scalar_type.clone(),
shape.clone(),
evq.range().into(),
with_values,
readopts,
scyqueue.clone(),
);
Box::pin(x)
@@ -84,7 +94,10 @@ pub async fn scylla_channel_event_stream(
item
}
},
Err(e) => Err(Error::with_msg_no_trace(format!("scyllaconn events error {e}"))),
Err(e) => Err(err::Error::with_msg_no_trace(format!(
"{}::scylla_channel_event_stream {e}",
module_path!()
))),
};
item
});