Move workspace crates into subfolder
This commit is contained in:
Dominik Werder
@@ -0,0 +1,197 @@
|
||||
use err::Error;
|
||||
use httpclient::url::Url;
|
||||
use netpod::log::*;
|
||||
use netpod::range::evrange::NanoRange;
|
||||
use netpod::timeunits::DAY;
|
||||
use netpod::AppendToUrl;
|
||||
use netpod::ByteOrder;
|
||||
use netpod::ChConf;
|
||||
use netpod::ChannelConfigQuery;
|
||||
use netpod::ChannelConfigResponse;
|
||||
use netpod::ChannelTypeConfigGen;
|
||||
use netpod::DtNano;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ScalarType;
|
||||
use netpod::SfChFetchInfo;
|
||||
use netpod::SfDbChannel;
|
||||
use netpod::Shape;
|
||||
use netpod::APP_JSON;
|
||||
use serde::Serialize;
|
||||
|
||||
const TEST_BACKEND: &str = "testbackend-00";
|
||||
|
||||
fn channel_config_test_backend(channel: SfDbChannel) -> Result<ChannelTypeConfigGen, Error> {
|
||||
let backend = channel.backend();
|
||||
let ret = if channel.name() == "scalar-i32-be" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
2,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::I32,
|
||||
Shape::Scalar,
|
||||
);
|
||||
ret
|
||||
} else if channel.name() == "wave-f64-be-n21" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
3,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::F64,
|
||||
Shape::Wave(21),
|
||||
);
|
||||
ret
|
||||
} else if channel.name() == "const-regular-scalar-i32-be" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
2,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::I32,
|
||||
Shape::Scalar,
|
||||
);
|
||||
ret
|
||||
} else if channel.name() == "test-gen-i32-dim0-v00" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
2,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::I32,
|
||||
Shape::Scalar,
|
||||
);
|
||||
ret
|
||||
} else if channel.name() == "test-gen-i32-dim0-v01" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
2,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::I32,
|
||||
Shape::Scalar,
|
||||
);
|
||||
ret
|
||||
} else if channel.name() == "test-gen-f64-dim1-v00" {
|
||||
let ret = SfChFetchInfo::new(
|
||||
backend,
|
||||
channel.name(),
|
||||
3,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::F64,
|
||||
Shape::Wave(21),
|
||||
);
|
||||
ret
|
||||
} else {
|
||||
error!("no test information");
|
||||
return Err(Error::with_msg_no_trace(format!("no test information"))
|
||||
.add_public_msg("No channel config for test channel {:?}"));
|
||||
};
|
||||
Ok(ChannelTypeConfigGen::SfDatabuffer(ret))
|
||||
}
|
||||
|
||||
pub async fn channel_config(
|
||||
range: NanoRange,
|
||||
channel: SfDbChannel,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
if channel.backend() == TEST_BACKEND {
|
||||
Ok(Some(channel_config_test_backend(channel)?))
|
||||
} else if ncc.node_config.cluster.scylla.is_some() {
|
||||
debug!("try to get ChConf for scylla type backend");
|
||||
let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
Ok(Some(ChannelTypeConfigGen::Scylla(ret)))
|
||||
} else if ncc.node.sf_databuffer.is_some() {
|
||||
debug!("channel_config channel {channel:?}");
|
||||
let k = disk::channelconfig::channel_config_best_match(range, channel.clone(), ncc)
|
||||
.await
|
||||
.map_err(|e| Error::from(e.to_string()))?;
|
||||
match k {
|
||||
Some(config) => {
|
||||
debug!("channel_config config {config:?}");
|
||||
let ret = SfChFetchInfo::new(
|
||||
config.channel.backend(),
|
||||
config.channel.name(),
|
||||
config.keyspace,
|
||||
config.time_bin_size,
|
||||
config.byte_order,
|
||||
config.scalar_type,
|
||||
config.shape,
|
||||
);
|
||||
let ret = ChannelTypeConfigGen::SfDatabuffer(ret);
|
||||
Ok(Some(ret))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
} else {
|
||||
return Err(
|
||||
Error::with_msg_no_trace(format!("no channel config for backend {}", channel.backend()))
|
||||
.add_public_msg(format!("no channel config for backend {}", channel.backend())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum ChannelConfigsGen {
|
||||
Scylla(ChConf),
|
||||
SfDatabuffer(disk::parse::channelconfig::ChannelConfigs),
|
||||
}
|
||||
|
||||
pub async fn channel_configs(channel: SfDbChannel, ncc: &NodeConfigCached) -> Result<ChannelConfigsGen, Error> {
|
||||
if channel.backend() == TEST_BACKEND {
|
||||
let ret = match channel_config_test_backend(channel)? {
|
||||
ChannelTypeConfigGen::Scylla(x) => ChannelConfigsGen::Scylla(x),
|
||||
ChannelTypeConfigGen::SfDatabuffer(x) => ChannelConfigsGen::SfDatabuffer(todo!()),
|
||||
};
|
||||
Ok(ret)
|
||||
} else if ncc.node_config.cluster.scylla.is_some() {
|
||||
debug!("try to get ChConf for scylla type backend");
|
||||
let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
Ok(ChannelConfigsGen::Scylla(ret))
|
||||
} else if ncc.node.sf_databuffer.is_some() {
|
||||
debug!("channel_config channel {channel:?}");
|
||||
let configs = disk::channelconfig::channel_configs(channel.clone(), ncc)
|
||||
.await
|
||||
.map_err(|e| Error::from(e.to_string()))?;
|
||||
Ok(ChannelConfigsGen::SfDatabuffer(configs))
|
||||
} else {
|
||||
return Err(
|
||||
Error::with_msg_no_trace(format!("no channel config for backend {}", channel.backend()))
|
||||
.add_public_msg(format!("no channel config for backend {}", channel.backend())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn http_get_channel_config(
|
||||
qu: ChannelConfigQuery,
|
||||
baseurl: Url,
|
||||
) -> Result<Option<ChannelConfigResponse>, Error> {
|
||||
let url = baseurl;
|
||||
let mut url = url.join("channel/config").unwrap();
|
||||
qu.append_to_url(&mut url);
|
||||
let res = httpclient::http_get(url, APP_JSON).await?;
|
||||
use httpclient::http::StatusCode;
|
||||
if res.head.status == StatusCode::NOT_FOUND {
|
||||
Ok(None)
|
||||
} else if res.head.status == StatusCode::OK {
|
||||
let ret: ChannelConfigResponse = serde_json::from_slice(&res.body)?;
|
||||
Ok(Some(ret))
|
||||
} else {
|
||||
let b = &res.body;
|
||||
let s = String::from_utf8_lossy(&b[0..b.len().min(256)]);
|
||||
Err(Error::with_msg_no_trace(format!(
|
||||
"http_get_channel_config {} {}",
|
||||
res.head.status, s
|
||||
)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
use crate::channelconfig::http_get_channel_config;
|
||||
use err::Error;
|
||||
use netpod::log::*;
|
||||
use netpod::range::evrange::SeriesRange;
|
||||
use netpod::ChConf;
|
||||
use netpod::ChannelConfigQuery;
|
||||
use netpod::ChannelConfigResponse;
|
||||
use netpod::ChannelTypeConfigGen;
|
||||
use netpod::DtNano;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::SfChFetchInfo;
|
||||
use netpod::SfDbChannel;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
fn decide_sf_ch_config_quorum(inp: Vec<ChannelConfigResponse>) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
let mut histo = BTreeMap::new();
|
||||
for item in inp {
|
||||
let item = match item {
|
||||
ChannelConfigResponse::SfDatabuffer(k) => ChannelTypeConfigGen::SfDatabuffer(SfChFetchInfo::new(
|
||||
k.backend,
|
||||
k.name,
|
||||
k.keyspace,
|
||||
DtNano::from_ms(k.timebinsize),
|
||||
k.byte_order,
|
||||
k.scalar_type,
|
||||
k.shape,
|
||||
)),
|
||||
ChannelConfigResponse::Daqbuf(k) => {
|
||||
ChannelTypeConfigGen::Scylla(ChConf::new(k.backend, k.series, k.scalar_type, k.shape, k.name))
|
||||
}
|
||||
};
|
||||
if histo.contains_key(&item) {
|
||||
*histo.get_mut(&item).unwrap() += 1;
|
||||
} else {
|
||||
histo.insert(item, 0u32);
|
||||
}
|
||||
}
|
||||
let mut v: Vec<_> = histo.into_iter().collect();
|
||||
v.sort_unstable_by_key(|x| x.1);
|
||||
match v.pop() {
|
||||
Some((x, _)) => Ok(Some(x)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn find_sf_ch_config_quorum(
|
||||
channel: SfDbChannel,
|
||||
range: SeriesRange,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<SfChFetchInfo>, Error> {
|
||||
let range = match range {
|
||||
SeriesRange::TimeRange(x) => x,
|
||||
SeriesRange::PulseRange(_) => return Err(Error::with_msg_no_trace("expect TimeRange")),
|
||||
};
|
||||
let mut all = Vec::new();
|
||||
for node in &ncc.node_config.cluster.nodes {
|
||||
// TODO add a baseurl function to struct Node
|
||||
let qu = ChannelConfigQuery {
|
||||
channel: channel.clone(),
|
||||
range: range.clone(),
|
||||
// TODO
|
||||
expand: false,
|
||||
};
|
||||
let res = tokio::time::timeout(Duration::from_millis(4000), http_get_channel_config(qu, node.baseurl()))
|
||||
.await
|
||||
.map_err(|_| Error::with_msg_no_trace("timeout"))??;
|
||||
all.push(res);
|
||||
}
|
||||
let all: Vec<_> = all.into_iter().filter_map(|x| x).collect();
|
||||
let qu = decide_sf_ch_config_quorum(all)?;
|
||||
match qu {
|
||||
Some(item) => match item {
|
||||
ChannelTypeConfigGen::Scylla(_) => Err(Error::with_msg_no_trace(
|
||||
"find_sf_ch_config_quorum not a sf-databuffer config",
|
||||
)),
|
||||
ChannelTypeConfigGen::SfDatabuffer(item) => Ok(Some(item)),
|
||||
},
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn find_config_basics_quorum(
|
||||
channel: SfDbChannel,
|
||||
range: SeriesRange,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Option<ChannelTypeConfigGen>, Error> {
|
||||
if let Some(_cfg) = &ncc.node.sf_databuffer {
|
||||
match find_sf_ch_config_quorum(channel, range, ncc).await? {
|
||||
Some(x) => Ok(Some(ChannelTypeConfigGen::SfDatabuffer(x))),
|
||||
None => Ok(None),
|
||||
}
|
||||
} else if let Some(_cfg) = &ncc.node_config.cluster.scylla {
|
||||
// TODO let called function allow to return None instead of error-not-found
|
||||
let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
Ok(Some(ChannelTypeConfigGen::Scylla(ret)))
|
||||
} else {
|
||||
Err(Error::with_msg_no_trace(
|
||||
"find_config_basics_quorum not supported backend",
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
use crate::scylla::scylla_channel_event_stream;
|
||||
use err::Error;
|
||||
use futures_util::Stream;
|
||||
use futures_util::StreamExt;
|
||||
use items_0::on_sitemty_data;
|
||||
use items_0::streamitem::sitem_data;
|
||||
use items_0::streamitem::LogItem;
|
||||
use items_0::streamitem::RangeCompletableItem;
|
||||
use items_0::streamitem::Sitemty;
|
||||
use items_0::streamitem::StreamItem;
|
||||
use items_0::streamitem::EVENT_QUERY_JSON_STRING_FRAME;
|
||||
use items_0::Events;
|
||||
use items_2::channelevents::ChannelEvents;
|
||||
use items_2::empty::empty_events_dyn_ev;
|
||||
use items_2::framable::EventQueryJsonStringFrame;
|
||||
use items_2::framable::Framable;
|
||||
use items_2::frame::decode_frame;
|
||||
use items_2::frame::make_term_frame;
|
||||
use items_2::inmem::InMemoryFrame;
|
||||
use netpod::histo::HistoLog2;
|
||||
use netpod::log::*;
|
||||
use netpod::NodeConfigCached;
|
||||
use query::api4::events::EventsSubQuery;
|
||||
use query::api4::events::Frame1Parts;
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use streams::frames::inmem::InMemoryFrameAsyncReadStream;
|
||||
use streams::generators::GenerateF64V00;
|
||||
use streams::generators::GenerateI32V00;
|
||||
use streams::generators::GenerateI32V01;
|
||||
use streams::transform::build_event_transform;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::tcp::OwnedReadHalf;
|
||||
use tokio::net::tcp::OwnedWriteHalf;
|
||||
use tokio::net::TcpStream;
|
||||
use tracing::Instrument;
|
||||
|
||||
const TEST_BACKEND: &str = "testbackend-00";
|
||||
|
||||
#[cfg(test)]
|
||||
mod test;
|
||||
|
||||
pub async fn events_service(node_config: NodeConfigCached) -> Result<(), Error> {
|
||||
let addr = format!("{}:{}", node_config.node.listen, node_config.node.port_raw);
|
||||
let lis = tokio::net::TcpListener::bind(addr).await?;
|
||||
loop {
|
||||
match lis.accept().await {
|
||||
Ok((stream, addr)) => {
|
||||
taskrun::spawn(events_conn_handler(stream, addr, node_config.clone()));
|
||||
}
|
||||
Err(e) => Err(e)?,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ConnErr {
|
||||
err: Error,
|
||||
#[allow(dead_code)]
|
||||
netout: OwnedWriteHalf,
|
||||
}
|
||||
|
||||
impl<E: Into<Error>> From<(E, OwnedWriteHalf)> for ConnErr {
|
||||
fn from((err, netout): (E, OwnedWriteHalf)) -> Self {
|
||||
Self {
|
||||
err: err.into(),
|
||||
netout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_channel_events_stream_data(
|
||||
subq: EventsSubQuery,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
|
||||
if subq.backend() == TEST_BACKEND {
|
||||
debug!("use test backend data {}", TEST_BACKEND);
|
||||
let node_count = ncc.node_config.cluster.nodes.len() as u64;
|
||||
let node_ix = ncc.ix as u64;
|
||||
let chn = subq.name();
|
||||
let range = subq.range().clone();
|
||||
let one_before = subq.transform().need_one_before_range();
|
||||
if chn == "test-gen-i32-dim0-v00" {
|
||||
Ok(Box::pin(GenerateI32V00::new(node_ix, node_count, range, one_before)))
|
||||
} else if chn == "test-gen-i32-dim0-v01" {
|
||||
Ok(Box::pin(GenerateI32V01::new(node_ix, node_count, range, one_before)))
|
||||
} else if chn == "test-gen-f64-dim1-v00" {
|
||||
Ok(Box::pin(GenerateF64V00::new(node_ix, node_count, range, one_before)))
|
||||
} else {
|
||||
let na: Vec<_> = chn.split("-").collect();
|
||||
if na.len() != 3 {
|
||||
Err(Error::with_msg_no_trace(format!(
|
||||
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
|
||||
)))
|
||||
} else {
|
||||
if na[0] != "inmem" {
|
||||
Err(Error::with_msg_no_trace(format!(
|
||||
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
|
||||
)))
|
||||
} else {
|
||||
let _range = subq.range().clone();
|
||||
if na[1] == "d0" {
|
||||
if na[2] == "i32" {
|
||||
//generator::generate_i32(node_ix, node_count, range)
|
||||
panic!()
|
||||
} else if na[2] == "f32" {
|
||||
//generator::generate_f32(node_ix, node_count, range)
|
||||
panic!()
|
||||
} else {
|
||||
Err(Error::with_msg_no_trace(format!(
|
||||
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
|
||||
)))
|
||||
}
|
||||
} else {
|
||||
Err(Error::with_msg_no_trace(format!(
|
||||
"make_channel_events_stream_data can not understand test channel name: {chn:?}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if let Some(scyconf) = &ncc.node_config.cluster.scylla {
|
||||
let cfg = subq.ch_conf().to_scylla()?;
|
||||
scylla_channel_event_stream(subq, cfg, scyconf, ncc).await
|
||||
} else if let Some(_) = &ncc.node.channel_archiver {
|
||||
let e = Error::with_msg_no_trace("archapp not built");
|
||||
Err(e)
|
||||
} else if let Some(_) = &ncc.node.archiver_appliance {
|
||||
let e = Error::with_msg_no_trace("archapp not built");
|
||||
Err(e)
|
||||
} else {
|
||||
let cfg = subq.ch_conf().to_sf_databuffer()?;
|
||||
Ok(disk::raw::conn::make_event_pipe(subq, cfg, ncc).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_channel_events_stream(
|
||||
subq: EventsSubQuery,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
|
||||
let empty = empty_events_dyn_ev(subq.ch_conf().scalar_type(), subq.ch_conf().shape())?;
|
||||
let empty = sitem_data(ChannelEvents::Events(empty));
|
||||
let stream = make_channel_events_stream_data(subq, ncc).await?;
|
||||
let ret = futures_util::stream::iter([empty]).chain(stream);
|
||||
let ret = Box::pin(ret);
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
async fn events_get_input_frames(netin: OwnedReadHalf) -> Result<Vec<InMemoryFrame>, Error> {
|
||||
let mut h = InMemoryFrameAsyncReadStream::new(netin, netpod::ByteSize::from_kb(1));
|
||||
let mut frames = Vec::new();
|
||||
while let Some(k) = h
|
||||
.next()
|
||||
.instrument(span!(Level::INFO, "events_conn_handler/query-input"))
|
||||
.await
|
||||
{
|
||||
match k {
|
||||
Ok(StreamItem::DataItem(item)) => {
|
||||
frames.push(item);
|
||||
}
|
||||
Ok(item) => {
|
||||
debug!("ignored incoming frame {:?}", item);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(frames)
|
||||
}
|
||||
|
||||
async fn events_parse_input_query(frames: Vec<InMemoryFrame>) -> Result<(EventsSubQuery,), Error> {
|
||||
if frames.len() != 1 {
|
||||
error!("{:?}", frames);
|
||||
error!("missing command frame len {}", frames.len());
|
||||
let e = Error::with_msg("missing command frame");
|
||||
return Err(e);
|
||||
}
|
||||
let query_frame = &frames[0];
|
||||
if query_frame.tyid() != EVENT_QUERY_JSON_STRING_FRAME {
|
||||
return Err(Error::with_msg("query frame wrong type"));
|
||||
}
|
||||
// TODO this does not need all variants of Sitemty.
|
||||
let qitem = match decode_frame::<Sitemty<EventQueryJsonStringFrame>>(query_frame) {
|
||||
Ok(k) => match k {
|
||||
Ok(k) => match k {
|
||||
StreamItem::DataItem(k) => match k {
|
||||
RangeCompletableItem::Data(k) => k,
|
||||
RangeCompletableItem::RangeComplete => return Err(Error::with_msg("bad query item")),
|
||||
},
|
||||
_ => return Err(Error::with_msg("bad query item")),
|
||||
},
|
||||
Err(e) => return Err(e),
|
||||
},
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let frame1: Frame1Parts = serde_json::from_str(&qitem.str()).map_err(|e| {
|
||||
let e = Error::with_msg_no_trace(format!("json parse error: {} inp {:?}", e, qitem.str()));
|
||||
error!("{e}");
|
||||
e
|
||||
})?;
|
||||
Ok(frame1.parts())
|
||||
}
|
||||
|
||||
async fn events_conn_handler_inner_try(
|
||||
stream: TcpStream,
|
||||
addr: SocketAddr,
|
||||
ncc: &NodeConfigCached,
|
||||
) -> Result<(), ConnErr> {
|
||||
let _ = addr;
|
||||
let (netin, mut netout) = stream.into_split();
|
||||
let frames = match events_get_input_frames(netin).await {
|
||||
Ok(x) => x,
|
||||
Err(e) => return Err((e, netout).into()),
|
||||
};
|
||||
let (evq,) = match events_parse_input_query(frames).await {
|
||||
Ok(x) => x,
|
||||
Err(e) => return Err((e, netout).into()),
|
||||
};
|
||||
info!("events_parse_input_query {evq:?}");
|
||||
if evq.create_errors_contains("nodenet_parse_query") {
|
||||
let e = Error::with_msg_no_trace("produced error on request nodenet_parse_query");
|
||||
return Err((e, netout).into());
|
||||
}
|
||||
let mut stream: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>> = if evq.is_event_blobs() {
|
||||
// TODO support event blobs as transform
|
||||
let fetch_info = match evq.ch_conf().to_sf_databuffer() {
|
||||
Ok(x) => x,
|
||||
Err(e) => return Err((e, netout).into()),
|
||||
};
|
||||
match disk::raw::conn::make_event_blobs_pipe(&evq, &fetch_info, ncc).await {
|
||||
Ok(stream) => {
|
||||
let stream = stream.map(|x| Box::new(x) as _);
|
||||
Box::pin(stream)
|
||||
}
|
||||
Err(e) => return Err((e, netout).into()),
|
||||
}
|
||||
} else {
|
||||
match make_channel_events_stream(evq.clone(), ncc).await {
|
||||
Ok(stream) => {
|
||||
if false {
|
||||
// TODO wasm example
|
||||
use wasmer::Value;
|
||||
let wasm = b"";
|
||||
let mut store = wasmer::Store::default();
|
||||
let module = wasmer::Module::new(&store, wasm).unwrap();
|
||||
let import_object = wasmer::imports! {};
|
||||
let instance = wasmer::Instance::new(&mut store, &module, &import_object).unwrap();
|
||||
let add_one = instance.exports.get_function("event_transform").unwrap();
|
||||
let result = add_one.call(&mut store, &[Value::I32(42)]).unwrap();
|
||||
assert_eq!(result[0], Value::I32(43));
|
||||
}
|
||||
let mut tr = match build_event_transform(evq.transform()) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
return Err((e, netout).into());
|
||||
}
|
||||
};
|
||||
let stream = stream.map(move |x| {
|
||||
let item = on_sitemty_data!(x, |x| {
|
||||
let x: Box<dyn Events> = Box::new(x);
|
||||
let x = tr.0.transform(x);
|
||||
Ok(StreamItem::DataItem(RangeCompletableItem::Data(x)))
|
||||
});
|
||||
Box::new(item) as Box<dyn Framable + Send>
|
||||
});
|
||||
Box::pin(stream)
|
||||
}
|
||||
Err(e) => {
|
||||
return Err((e, netout).into());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut buf_len_histo = HistoLog2::new(5);
|
||||
while let Some(item) = stream.next().await {
|
||||
let item = item.make_frame();
|
||||
match item {
|
||||
Ok(buf) => {
|
||||
buf_len_histo.ingest(buf.len() as u32);
|
||||
match netout.write_all(&buf).await {
|
||||
Ok(()) => {
|
||||
// TODO collect timing information and send as summary in a stats item.
|
||||
// TODO especially collect a distribution over the buf lengths that were send.
|
||||
// TODO we want to see a reasonable batch size.
|
||||
}
|
||||
Err(e) => return Err((e, netout))?,
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("events_conn_handler_inner_try sees error in stream: {e:?}");
|
||||
return Err((e, netout))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
let item = LogItem {
|
||||
node_ix: ncc.ix as _,
|
||||
level: Level::INFO,
|
||||
msg: format!("buf_len_histo: {:?}", buf_len_histo),
|
||||
};
|
||||
let item: Sitemty<ChannelEvents> = Ok(StreamItem::Log(item));
|
||||
let buf = match item.make_frame() {
|
||||
Ok(k) => k,
|
||||
Err(e) => return Err((e, netout))?,
|
||||
};
|
||||
match netout.write_all(&buf).await {
|
||||
Ok(()) => (),
|
||||
Err(e) => return Err((e, netout))?,
|
||||
}
|
||||
}
|
||||
let buf = match make_term_frame() {
|
||||
Ok(k) => k,
|
||||
Err(e) => return Err((e, netout))?,
|
||||
};
|
||||
match netout.write_all(&buf).await {
|
||||
Ok(()) => (),
|
||||
Err(e) => return Err((e, netout))?,
|
||||
}
|
||||
match netout.flush().await {
|
||||
Ok(()) => (),
|
||||
Err(e) => return Err((e, netout))?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn events_conn_handler_inner(
|
||||
stream: TcpStream,
|
||||
addr: SocketAddr,
|
||||
node_config: &NodeConfigCached,
|
||||
) -> Result<(), Error> {
|
||||
match events_conn_handler_inner_try(stream, addr, node_config).await {
|
||||
Ok(_) => (),
|
||||
Err(ce) => {
|
||||
let mut out = ce.netout;
|
||||
let item: Sitemty<ChannelEvents> = Err(ce.err);
|
||||
let buf = Framable::make_frame(&item)?;
|
||||
out.write_all(&buf).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn events_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: NodeConfigCached) -> Result<(), Error> {
|
||||
let span1 = span!(Level::INFO, "events_conn_handler");
|
||||
let r = events_conn_handler_inner(stream, addr, &node_config)
|
||||
.instrument(span1)
|
||||
.await;
|
||||
match r {
|
||||
Ok(k) => Ok(k),
|
||||
Err(e) => {
|
||||
error!("events_conn_handler sees error: {:?}", e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
use crate::conn::events_conn_handler;
|
||||
use crate::conn::Frame1Parts;
|
||||
use err::Error;
|
||||
use futures_util::StreamExt;
|
||||
use items_0::streamitem::sitem_data;
|
||||
use items_0::streamitem::Sitemty;
|
||||
use items_0::streamitem::StreamItem;
|
||||
use items_0::streamitem::ERROR_FRAME_TYPE_ID;
|
||||
use items_0::streamitem::ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID;
|
||||
use items_0::streamitem::LOG_FRAME_TYPE_ID;
|
||||
use items_0::streamitem::STATS_FRAME_TYPE_ID;
|
||||
use items_2::channelevents::ChannelEvents;
|
||||
use items_2::framable::EventQueryJsonStringFrame;
|
||||
use items_2::framable::Framable;
|
||||
use items_2::frame::decode_frame;
|
||||
use netpod::range::evrange::NanoRange;
|
||||
use netpod::timeunits::DAY;
|
||||
use netpod::timeunits::SEC;
|
||||
use netpod::ByteOrder;
|
||||
use netpod::Cluster;
|
||||
use netpod::Database;
|
||||
use netpod::DtNano;
|
||||
use netpod::FileIoBufferSize;
|
||||
use netpod::Node;
|
||||
use netpod::NodeConfig;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ScalarType;
|
||||
use netpod::SfChFetchInfo;
|
||||
use netpod::SfDatabuffer;
|
||||
use netpod::Shape;
|
||||
use query::api4::events::EventsSubQuery;
|
||||
use query::api4::events::EventsSubQuerySelect;
|
||||
use query::api4::events::EventsSubQuerySettings;
|
||||
use query::transform::TransformQuery;
|
||||
use streams::frames::inmem::InMemoryFrameAsyncReadStream;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
const TEST_BACKEND: &str = "testbackend-00";
|
||||
|
||||
#[test]
|
||||
fn raw_data_00() {
|
||||
let fut = async {
|
||||
let lis = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let mut con = TcpStream::connect(lis.local_addr().unwrap()).await.unwrap();
|
||||
let (client, addr) = lis.accept().await.unwrap();
|
||||
let cfg = NodeConfigCached {
|
||||
node_config: NodeConfig {
|
||||
name: "node_name_dummy".into(),
|
||||
cluster: Cluster {
|
||||
backend: TEST_BACKEND.into(),
|
||||
nodes: Vec::new(),
|
||||
database: Database {
|
||||
name: "".into(),
|
||||
host: "".into(),
|
||||
port: 5432,
|
||||
user: "".into(),
|
||||
pass: "".into(),
|
||||
},
|
||||
run_map_pulse_task: false,
|
||||
is_central_storage: false,
|
||||
file_io_buffer_size: FileIoBufferSize(1024 * 8),
|
||||
scylla: None,
|
||||
cache_scylla: None,
|
||||
},
|
||||
},
|
||||
node: Node {
|
||||
host: "empty".into(),
|
||||
listen: "listen_dummy".into(),
|
||||
port: 9090,
|
||||
port_raw: 9090,
|
||||
cache_base_path: "".into(),
|
||||
sf_databuffer: Some(SfDatabuffer {
|
||||
data_base_path: "/home/dominik/daqbuffer-testdata/databuffer/node00".into(),
|
||||
ksprefix: "ks".into(),
|
||||
splits: None,
|
||||
}),
|
||||
archiver_appliance: None,
|
||||
channel_archiver: None,
|
||||
prometheus_api_bind: None,
|
||||
},
|
||||
ix: 0,
|
||||
};
|
||||
let range = NanoRange {
|
||||
beg: SEC,
|
||||
end: SEC * 10,
|
||||
};
|
||||
let fetch_info = SfChFetchInfo::new(
|
||||
TEST_BACKEND,
|
||||
"scalar-i32",
|
||||
2,
|
||||
DtNano::from_ns(DAY),
|
||||
ByteOrder::Big,
|
||||
ScalarType::I32,
|
||||
Shape::Scalar,
|
||||
);
|
||||
let select = EventsSubQuerySelect::new(fetch_info.into(), range.into(), TransformQuery::default_events());
|
||||
let settings = EventsSubQuerySettings::default();
|
||||
let qu = EventsSubQuery::from_parts(select, settings);
|
||||
let frame1 = Frame1Parts::new(qu.clone());
|
||||
let query = EventQueryJsonStringFrame(serde_json::to_string(&frame1).unwrap());
|
||||
let frame = sitem_data(query).make_frame()?;
|
||||
let jh = taskrun::spawn(events_conn_handler(client, addr, cfg));
|
||||
con.write_all(&frame).await.unwrap();
|
||||
eprintln!("written");
|
||||
con.shutdown().await.unwrap();
|
||||
eprintln!("shut down");
|
||||
|
||||
let mut frames = InMemoryFrameAsyncReadStream::new(con, qu.inmem_bufcap());
|
||||
while let Some(frame) = frames.next().await {
|
||||
match frame {
|
||||
Ok(frame) => match frame {
|
||||
StreamItem::DataItem(k) => {
|
||||
eprintln!("{k:?}");
|
||||
if k.tyid() == ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID {
|
||||
} else if k.tyid() == ERROR_FRAME_TYPE_ID {
|
||||
} else if k.tyid() == LOG_FRAME_TYPE_ID {
|
||||
} else if k.tyid() == STATS_FRAME_TYPE_ID {
|
||||
} else {
|
||||
panic!("unexpected frame type id {:x}", k.tyid());
|
||||
}
|
||||
let item: Sitemty<ChannelEvents> = decode_frame(&k).unwrap();
|
||||
eprintln!("decoded: {:?}", item);
|
||||
}
|
||||
StreamItem::Log(_) => todo!(),
|
||||
StreamItem::Stats(_) => todo!(),
|
||||
},
|
||||
Err(e) => {
|
||||
panic!("{e:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
jh.await.unwrap().unwrap();
|
||||
Ok::<_, Error>(())
|
||||
};
|
||||
taskrun::run(fut).unwrap();
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod channelconfig;
|
||||
pub mod configquorum;
|
||||
pub mod conn;
|
||||
pub mod scylla;
|
||||
@@ -0,0 +1,80 @@
|
||||
use err::Error;
|
||||
use futures_util::Stream;
|
||||
use futures_util::StreamExt;
|
||||
use items_0::streamitem::RangeCompletableItem;
|
||||
use items_0::streamitem::Sitemty;
|
||||
use items_0::streamitem::StreamItem;
|
||||
use items_2::channelevents::ChannelEvents;
|
||||
use netpod::log::*;
|
||||
use netpod::ChConf;
|
||||
use netpod::NodeConfigCached;
|
||||
use netpod::ScyllaConfig;
|
||||
use query::api4::events::EventsSubQuery;
|
||||
use std::pin::Pin;
|
||||
|
||||
pub async fn scylla_channel_event_stream(
|
||||
evq: EventsSubQuery,
|
||||
chconf: ChConf,
|
||||
scyco: &ScyllaConfig,
|
||||
_ncc: &NodeConfigCached,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
|
||||
// TODO depends in general on the query
|
||||
// TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
|
||||
let do_one_before_range = false;
|
||||
// TODO use better builder pattern with shortcuts for production and dev defaults
|
||||
let scy = scyllaconn::create_scy_session(scyco).await?;
|
||||
let series = chconf.series();
|
||||
let scalar_type = chconf.scalar_type();
|
||||
let shape = chconf.shape();
|
||||
let do_test_stream_error = false;
|
||||
let with_values = evq.need_value_data();
|
||||
debug!("Make EventsStreamScylla for {series:?} {scalar_type:?} {shape:?}");
|
||||
let stream = scyllaconn::events::EventsStreamScylla::new(
|
||||
series,
|
||||
evq.range().into(),
|
||||
do_one_before_range,
|
||||
scalar_type.clone(),
|
||||
shape.clone(),
|
||||
with_values,
|
||||
scy,
|
||||
do_test_stream_error,
|
||||
);
|
||||
let stream = stream
|
||||
.map(move |item| match &item {
|
||||
Ok(k) => match k {
|
||||
ChannelEvents::Events(k) => {
|
||||
let n = k.len();
|
||||
let d = evq.event_delay();
|
||||
(item, n, d.clone())
|
||||
}
|
||||
ChannelEvents::Status(_) => (item, 1, None),
|
||||
},
|
||||
Err(_) => (item, 1, None),
|
||||
})
|
||||
.then(|(item, n, d)| async move {
|
||||
if let Some(d) = d {
|
||||
warn!("sleep {} times {:?}", n, d);
|
||||
tokio::time::sleep(d.saturating_mul(n as _)).await;
|
||||
}
|
||||
item
|
||||
})
|
||||
.map(|item| {
|
||||
let item = match item {
|
||||
Ok(item) => match item {
|
||||
ChannelEvents::Events(item) => {
|
||||
let item = ChannelEvents::Events(item);
|
||||
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
|
||||
item
|
||||
}
|
||||
ChannelEvents::Status(item) => {
|
||||
let item = ChannelEvents::Status(item);
|
||||
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
|
||||
item
|
||||
}
|
||||
},
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
item
|
||||
});
|
||||
Ok(Box::pin(stream))
|
||||
}
|
||||
Reference in New Issue
Block a user