daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP checks

Browse Source
This commit is contained in:
Dominik Werder
2023-04-05 12:00:18 +02:00
parent 7c9085fe5b
commit 81298b16df
39 changed files with 892 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
nodenet/src/channelconfig.rs
View File

@@ -72,7 +72,14 @@ pub async fn channel_config(range: NanoRange, channel: Channel, ncc: &NodeConfig
info!("channel_config BEFORE {channel:?}");
info!("try to get ChConf for sf-databuffer type backend");
// TODO in the future we should not need this:
let channel = sf_databuffer_fetch_channel_by_series(channel, ncc).await?;
let mut channel = sf_databuffer_fetch_channel_by_series(channel, ncc).await?;
if channel.series.is_none() {
let pgclient = dbconn::create_connection(&ncc.node_config.cluster.database).await?;
let pgclient = std::sync::Arc::new(pgclient);
let series = dbconn::find_series_sf_databuffer(&channel, pgclient).await?;
channel.series = Some(series);
}
let channel = channel;
info!("channel_config AFTER {channel:?}");
let c1 = disk::channelconfig::config(range, channel.clone(), ncc).await?;
info!("channel_config THEN {c1:?}");

64
nodenet/src/conn.rs
View File

@@ -1,3 +1,5 @@
pub mod generator;
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
@@ -7,8 +9,6 @@ use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::streamitem::EVENT_QUERY_JSON_STRING_FRAME;
use items_0::Appendable;
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use items_2::framable::EventQueryJsonStringFrame;
use items_2::framable::Framable;
@@ -17,7 +17,6 @@ use items_2::frame::make_term_frame;
use items_2::inmem::InMemoryFrame;
use netpod::histo::HistoLog2;
use netpod::log::*;
use netpod::AggKind;
use netpod::ChConf;
use netpod::NodeConfigCached;
use netpod::PerfOpts;
@@ -72,36 +71,37 @@ async fn make_channel_events_stream(
info!("nodenet::conn::make_channel_events_stream");
if evq.channel().backend() == "test-inmem" {
warn!("TEST BACKEND DATA");
use netpod::timeunits::MS;
let node_count = node_config.node_config.cluster.nodes.len();
let node_ix = node_config.ix;
if evq.channel().name() == "inmem-d0-i32" {
let mut item = items_2::eventsdim0::EventsDim0::<i32>::empty();
let td = MS * 10;
for i in 0..20 {
let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
let pulse = 1 + node_ix as u64 + node_count as u64 * i;
item.push(ts, pulse, pulse as _);
}
let item = ChannelEvents::Events(Box::new(item) as _);
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
let stream = futures_util::stream::iter([item]);
Ok(Box::pin(stream))
} else if evq.channel().name() == "inmem-d0-f32" {
let mut item = items_2::eventsdim0::EventsDim0::<f32>::empty();
let td = MS * 10;
for i in 0..20 {
let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
let pulse = 1 + node_ix as u64 + node_count as u64 * i;
item.push(ts, pulse, ts as _);
}
let item = ChannelEvents::Events(Box::new(item) as _);
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
let stream = futures_util::stream::iter([item]);
Ok(Box::pin(stream))
let node_count = node_config.node_config.cluster.nodes.len() as u64;
let node_ix = node_config.ix as u64;
let chn = evq.channel().name();
let na: Vec<_> = chn.split("-").collect();
if na.len() != 3 {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
} else {
let stream = futures_util::stream::empty();
Ok(Box::pin(stream))
if na[0] != "inmem" {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
} else {
let range = evq.range().clone();
if na[1] == "d0" {
if na[2] == "i32" {
generator::generate_i32(node_ix, node_count, range)
} else if na[2] == "f32" {
generator::generate_f32(node_ix, node_count, range)
} else {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
}
} else {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
}
}
}
} else if let Some(scyconf) = &node_config.node_config.cluster.scylla {
scylla_channel_event_stream(evq, chconf, scyconf, node_config).await

72
nodenet/src/conn/generator.rs Normal file
View File

@@ -0,0 +1,72 @@
use err::Error;
use futures_util::Stream;
use items_0::container::ByteEstimate;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
use items_0::Empty;
use items_0::WithLen;
use items_2::channelevents::ChannelEvents;
use netpod::log::*;
use netpod::range::evrange::SeriesRange;
use netpod::timeunits::MS;
use std::pin::Pin;
pub fn generate_i32(
node_ix: u64,
node_count: u64,
range: SeriesRange,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
type T = i32;
let mut items = Vec::new();
match range {
SeriesRange::TimeRange(range) => {
let mut item = items_2::eventsdim0::EventsDim0::empty();
let td = MS * 1000;
let mut ts = (range.beg / td + node_ix) * td;
loop {
if ts >= range.end {
break;
}
let pulse = ts;
item.push(ts, pulse, pulse as T);
ts += td * node_count as u64;
if item.byte_estimate() > 200 {
let w = ChannelEvents::Events(Box::new(item) as _);
let w = Ok::<_, Error>(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(w);
item = items_2::eventsdim0::EventsDim0::empty();
}
}
if item.len() != 0 {
let w = ChannelEvents::Events(Box::new(item) as _);
let w = Ok::<_, Error>(StreamItem::DataItem(RangeCompletableItem::Data(w)));
items.push(w);
}
}
SeriesRange::PulseRange(_) => {
error!("TODO generate test data by pulse id range");
}
}
let stream = futures_util::stream::iter(items);
Ok(Box::pin(stream))
}
pub fn generate_f32(
node_ix: u64,
node_count: u64,
range: SeriesRange,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
let mut item = items_2::eventsdim0::EventsDim0::<f32>::empty();
let td = MS * 10;
for i in 0..20 {
let ts = MS * 17 + td * node_ix as u64 + td * node_count as u64 * i;
let pulse = 1 + node_ix as u64 + node_count as u64 * i;
item.push(ts, pulse, ts as _);
}
let item = ChannelEvents::Events(Box::new(item) as _);
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)));
let stream = futures_util::stream::iter([item]);
Ok(Box::pin(stream))
}