daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP adding cbor stream test

Browse Source
This commit is contained in:
Dominik Werder
2023-12-18 15:53:33 +01:00
parent 31afee0893
commit a8479b2c8d
16 changed files with 407 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
crates/nodenet/src/client.rs Normal file
View File

@@ -0,0 +1,91 @@
use err::Error;
use futures_util::Future;
use http::header;
use http::Method;
use http::Request;
use httpclient::body_bytes;
use httpclient::http;
use httpclient::hyper::StatusCode;
use httpclient::hyper::Uri;
use items_0::streamitem::sitem_data;
use items_2::framable::Framable;
use netpod::log::*;
use netpod::Cluster;
use netpod::ReqCtx;
use netpod::APP_OCTET;
use query::api4::events::EventsSubQuery;
use std::pin::Pin;
use streams::frames::inmem::BoxedBytesStream;
use streams::tcprawclient::make_node_command_frame;
use streams::tcprawclient::OpenBoxedBytesStreams;
async fn open_bytes_data_streams_http(
subq: EventsSubQuery,
ctx: ReqCtx,
cluster: Cluster,
) -> Result<Vec<BoxedBytesStream>, Error> {
let frame1 = make_node_command_frame(subq.clone())?;
let mut streams = Vec::new();
for node in &cluster.nodes {
let item = sitem_data(frame1.clone());
let buf = item.make_frame()?;
let url = node.baseurl().join("/api/4/private/eventdata/frames").unwrap();
debug!("open_event_data_streams_http post {url}");
let uri: Uri = url.as_str().parse().unwrap();
let req = Request::builder()
.method(Method::POST)
.uri(&uri)
.header(header::HOST, uri.host().unwrap())
.header(header::ACCEPT, APP_OCTET)
.header(ctx.header_name(), ctx.header_value())
.body(body_bytes(buf))
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
let mut client = httpclient::connect_client(req.uri()).await?;
let res = client
.send_request(req)
.await
.map_err(|e| Error::with_msg_no_trace(e.to_string()))?;
if res.status() != StatusCode::OK {
error!("Server error {:?}", res);
let (head, body) = res.into_parts();
let buf = httpclient::read_body_bytes(body).await?;
let s = String::from_utf8_lossy(&buf);
return Err(Error::with_msg(format!(
concat!(
"Server error {:?}\n",
"---------------------- message from http body:\n",
"{}\n",
"---------------------- end of http body",
),
head, s
)));
}
let (_head, body) = res.into_parts();
let stream = Box::pin(httpclient::IncomingStream::new(body)) as BoxedBytesStream;
debug!("open_event_data_streams_http done {url}");
streams.push(Box::pin(stream) as _);
}
Ok(streams)
}
pub struct OpenBoxedBytesViaHttp {
cluster: Cluster,
}
impl OpenBoxedBytesViaHttp {
pub fn new(cluster: Cluster) -> Self {
Self { cluster }
}
}
impl OpenBoxedBytesStreams for OpenBoxedBytesViaHttp {
fn open(
&self,
subq: EventsSubQuery,
ctx: ReqCtx,
) -> Pin<Box<dyn Future<Output = Result<Vec<BoxedBytesStream>, Error>> + Send>> {
let fut = open_bytes_data_streams_http(subq, ctx, self.cluster.clone());
Box::pin(fut)
}
}

58
crates/nodenet/src/conn.rs
View File

@@ -27,11 +27,10 @@ use query::api4::events::EventsSubQuery;
use query::api4::events::Frame1Parts;
use std::net::SocketAddr;
use std::pin::Pin;
use streams::frames::inmem::BoxedBytesStream;
use streams::frames::inmem::InMemoryFrameStream;
use streams::frames::inmem::TcpReadAsBytes;
use streams::generators::GenerateF64V00;
use streams::generators::GenerateI32V00;
use streams::generators::GenerateI32V01;
use streams::tcprawclient::TEST_BACKEND;
use streams::transform::build_event_transform;
use taskrun::tokio;
use tokio::io::AsyncWriteExt;
@@ -39,8 +38,6 @@ use tokio::net::tcp::OwnedWriteHalf;
use tokio::net::TcpStream;
use tracing::Instrument;
const TEST_BACKEND: &str = "testbackend-00";
#[cfg(test)]
mod test;
@@ -81,51 +78,9 @@ async fn make_channel_events_stream_data(
ncc: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
if subq.backend() == TEST_BACKEND {
debug!("use test backend data {}", TEST_BACKEND);
let node_count = ncc.node_config.cluster.nodes.len() as u64;
let node_ix = ncc.ix as u64;
let chn = subq.name();
let range = subq.range().clone();
let one_before = subq.transform().need_one_before_range();
if chn == "test-gen-i32-dim0-v00" {
Ok(Box::pin(GenerateI32V00::new(node_ix, node_count, range, one_before)))
} else if chn == "test-gen-i32-dim0-v01" {
Ok(Box::pin(GenerateI32V01::new(node_ix, node_count, range, one_before)))
} else if chn == "test-gen-f64-dim1-v00" {
Ok(Box::pin(GenerateF64V00::new(node_ix, node_count, range, one_before)))
} else {
let na: Vec<_> = chn.split("-").collect();
if na.len() != 3 {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
} else {
if na[0] != "inmem" {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
} else {
let _range = subq.range().clone();
if na[1] == "d0" {
if na[2] == "i32" {
//generator::generate_i32(node_ix, node_count, range)
panic!()
} else if na[2] == "f32" {
//generator::generate_f32(node_ix, node_count, range)
panic!()
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
}
} else {
Err(Error::with_msg_no_trace(format!(
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
)))
}
}
}
}
streams::generators::make_test_channel_events_stream_data(subq, node_count, node_ix)
} else if let Some(scyconf) = &ncc.node_config.cluster.scylla {
let cfg = subq.ch_conf().to_scylla()?;
scylla_channel_event_stream(subq, cfg, scyconf, ncc).await
@@ -154,12 +109,10 @@ async fn make_channel_events_stream(
Ok(ret)
}
pub type BytesStreamBox = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
pub async fn create_response_bytes_stream(
evq: EventsSubQuery,
ncc: &NodeConfigCached,
) -> Result<BytesStreamBox, Error> {
) -> Result<BoxedBytesStream, Error> {
debug!(
"create_response_bytes_stream {:?} {:?}",
evq.ch_conf().scalar_type(),
@@ -180,9 +133,8 @@ pub async fn create_response_bytes_stream(
let ret = Box::pin(stream);
Ok(ret)
} else {
let stream = make_channel_events_stream(evq.clone(), reqctx, ncc).await?;
let mut tr = build_event_transform(evq.transform())?;
let stream = make_channel_events_stream(evq, reqctx, ncc).await?;
let stream = stream.map(move |x| {
on_sitemty_data!(x, |x: ChannelEvents| {
match x {

1
crates/nodenet/src/lib.rs
View File

@@ -1,4 +1,5 @@
pub mod channelconfig;
pub mod client;
pub mod configquorum;
pub mod conn;
pub mod scylla;