daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2023-04-25 09:08:14 +02:00
parent 95af6c359c
commit 498ff3612b
36 changed files with 1500 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
disk/src/channelconfig.rs
View File

@@ -1,3 +1,4 @@
use crate::SfDbChConf;
use err::Error;
use netpod::range::evrange::NanoRange;
use netpod::Channel;
@@ -7,10 +8,8 @@ use parse::channelconfig::read_local_config;
use parse::channelconfig::ChannelConfigs;
use parse::channelconfig::MatchingConfigEntry;
use crate::SfDbChConf;
pub async fn config(range: NanoRange, channel: Channel, node_config: &NodeConfigCached) -> Result<SfDbChConf, Error> {
let channel_configs = read_local_config(channel.clone(), node_config.node.clone()).await?;
let channel_configs = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(&range, &channel_configs) {
Ok(k) => k,
Err(e) => return Err(e)?,
@@ -48,5 +47,5 @@ pub async fn config(range: NanoRange, channel: Channel, node_config: &NodeConfig
}
pub async fn configs(channel: Channel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
read_local_config(channel.clone(), node_config.node.clone()).await
read_local_config(channel.clone(), node_config.clone()).await
}

4
disk/src/dataopen.rs
View File

@@ -19,6 +19,8 @@ use tokio::io::AsyncSeekExt;
use tokio::io::ErrorKind;
use tokio::io::SeekFrom;
const BACKEND: &str = "testbackend-00";
pub struct Positioned {
pub file: OpenedFile,
pub found: bool,
@@ -821,7 +823,7 @@ mod test {
end: DAY + HOUR * 8,
};
let chn = netpod::Channel {
backend: "test-disk-databuffer".into(),
backend: BACKEND.into(),
name: "scalar-i32-be".into(),
series: None,
};

4
disk/src/eventblobs.rs
View File

@@ -289,9 +289,11 @@ mod test {
use netpod::TsNano;
use streams::rangefilter2::RangeFilter2;
const BACKEND: &str = "testbackend-00";
fn read_expanded_for_range(range: NanoRange, nodeix: usize) -> Result<(usize, Vec<u64>), Error> {
let chn = netpod::Channel {
backend: "test-disk-databuffer".into(),
backend: BACKEND.into(),
name: "scalar-i32-be".into(),
series: None,
};

4
disk/src/eventchunker.rs
View File

@@ -575,11 +575,13 @@ mod test {
//use netpod::timeunits::*;
//use netpod::{ByteSize, Nanos};
//const TEST_BACKEND: &str = "testbackend-00";
/*
#[test]
fn read_expanded_for_range(range: netpod::NanoRange, nodeix: usize) -> Result<(usize, usize), Error> {
let chn = netpod::Channel {
backend: "testbackend".into(),
backend: TEST_BACKEND.into(),
name: "scalar-i32-be".into(),
};
// TODO read config from disk.

4
disk/src/gen.rs
View File

@@ -20,8 +20,10 @@ use tokio::fs::File;
use tokio::fs::OpenOptions;
use tokio::io::AsyncWriteExt;
const BACKEND: &str = "testbackend-00";
pub async fn gen_test_data() -> Result<(), Error> {
let backend = String::from("test-disk-databuffer");
let backend = String::from(BACKEND);
let homedir = std::env::var("HOME").unwrap();
let data_base_path = PathBuf::from(homedir).join("daqbuffer-testdata").join("databuffer");
let ksprefix = String::from("ks");

1
disk/src/raw.rs
View File

@@ -1 +1,2 @@
pub mod conn;
pub mod generated;

64
disk/src/raw/conn.rs
View File

@@ -1,5 +1,7 @@
use crate::eventblobs::EventChunkerMultifile;
use crate::eventchunker::EventChunkerConf;
use crate::raw::generated::EventBlobsGeneratorI32Test00;
use crate::raw::generated::EventBlobsGeneratorI32Test01;
use crate::SfDbChConf;
use err::Error;
use futures_util::stream;
@@ -25,6 +27,8 @@ use parse::channelconfig::MatchingConfigEntry;
use query::api4::events::PlainEventsQuery;
use std::pin::Pin;
const TEST_BACKEND: &str = "testbackend-00";
fn make_num_pipeline_stream_evs(
chconf: ChConf,
agg_kind: AggKind,
@@ -130,7 +134,7 @@ pub async fn get_applicable_entry(
node_config: &NodeConfigCached,
) -> Result<ConfigEntry, Error> {
info!("---------- disk::raw::conn::get_applicable_entry");
let channel_config = read_local_config(channel.clone(), node_config.node.clone()).await?;
let channel_config = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(range, &channel_config) {
Ok(k) => k,
Err(e) => return Err(e)?,
@@ -248,11 +252,10 @@ pub fn make_remote_event_blobs_stream(
Ok(event_blobs)
}
pub async fn make_event_blobs_pipe(
pub async fn make_event_blobs_pipe_real(
evq: &PlainEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
info!("make_event_blobs_pipe {evq:?}");
if false {
match dbconn::channel_exists(evq.channel(), &node_config).await {
Ok(_) => (),
@@ -312,3 +315,58 @@ pub async fn make_event_blobs_pipe(
};
Ok(pipe)
}
pub async fn make_event_blobs_pipe_test(
evq: &PlainEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
warn!("GENERATE INMEM TEST DATA");
let node_count = node_config.node_config.cluster.nodes.len() as u64;
let node_ix = node_config.ix as u64;
let chn = evq.channel().name();
let range = evq.range().clone();
if chn == "test-gen-i32-dim0-v00" {
Ok(Box::pin(EventBlobsGeneratorI32Test00::new(node_ix, node_count, range)))
} else if chn == "test-gen-i32-dim0-v01" {
Ok(Box::pin(EventBlobsGeneratorI32Test01::new(node_ix, node_count, range)))
} else {
let na: Vec<_> = chn.split("-").collect();
if na.len() != 3 {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
} else {
if na[0] != "inmem" {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
} else {
if na[1] == "d0" {
if na[2] == "i32" {
Ok(Box::pin(EventBlobsGeneratorI32Test00::new(node_ix, node_count, range)))
} else {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
}
} else {
Err(Error::with_msg_no_trace(format!(
"can not understand test channel name: {chn:?}"
)))
}
}
}
}
}
pub async fn make_event_blobs_pipe(
evq: &PlainEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
info!("make_event_blobs_pipe {evq:?}");
if evq.channel().backend() == TEST_BACKEND {
make_event_blobs_pipe_test(evq, node_config).await
} else {
make_event_blobs_pipe_real(evq, node_config).await
}
}

223
disk/src/raw/generated.rs Normal file
View File

@@ -0,0 +1,223 @@
use futures_util::Future;
use futures_util::FutureExt;
use futures_util::Stream;
use items_0::container::ByteEstimate;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Empty;
use items_2::eventfull::EventFull;
use netpod::range::evrange::SeriesRange;
use netpod::timeunits::MS;
use netpod::ScalarType;
use netpod::Shape;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
use std::time::Duration;
pub trait TypedGenerator {
type RustScalar;
}
pub struct EventBlobsGeneratorI32Test00 {
ts: u64,
dts: u64,
tsend: u64,
#[allow(unused)]
c1: u64,
scalar_type: ScalarType,
be: bool,
shape: Shape,
timeout: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
done: bool,
done_range_final: bool,
}
impl TypedGenerator for EventBlobsGeneratorI32Test00 {
type RustScalar = i32;
}
impl EventBlobsGeneratorI32Test00 {
pub fn new(node_ix: u64, node_count: u64, range: SeriesRange) -> Self {
let range = match range {
SeriesRange::TimeRange(k) => k,
SeriesRange::PulseRange(_) => todo!(),
};
let dts = MS * 1000 * node_count as u64;
let ts = (range.beg / dts + node_ix) * dts;
let tsend = range.end;
Self {
ts,
dts,
tsend,
c1: 0,
scalar_type: ScalarType::I32,
be: true,
shape: Shape::Scalar,
timeout: None,
done: false,
done_range_final: false,
}
}
fn make_batch(&mut self) -> Sitemty<EventFull> {
// TODO should not repeat self type name
type T = <EventBlobsGeneratorI32Test00 as TypedGenerator>::RustScalar;
let mut item = EventFull::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 200 {
break;
}
let pulse = ts;
let value = (ts / (MS * 100) % 1000) as T;
item.add_event(
ts,
pulse,
Some(value.to_be_bytes().to_vec()),
None,
self.scalar_type.clone(),
self.be,
self.shape.clone(),
None,
);
ts += self.dts;
}
self.ts = ts;
let w = sitem_data(item);
w
}
}
impl Stream for EventBlobsGeneratorI32Test00 {
type Item = Sitemty<EventFull>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.done_range_final {
Ready(None)
} else if self.ts >= self.tsend {
self.done = true;
self.done_range_final = true;
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))))
} else if false {
// To use the generator without throttling, use this scope
Ready(Some(self.make_batch()))
} else if let Some(fut) = self.timeout.as_mut() {
match fut.poll_unpin(cx) {
Ready(()) => {
self.timeout = None;
Ready(Some(self.make_batch()))
}
Pending => Pending,
}
} else {
self.timeout = Some(Box::pin(tokio::time::sleep(Duration::from_millis(2))));
continue;
};
}
}
}
pub struct EventBlobsGeneratorI32Test01 {
ts: u64,
dts: u64,
tsend: u64,
#[allow(unused)]
c1: u64,
scalar_type: ScalarType,
be: bool,
shape: Shape,
timeout: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
done: bool,
done_range_final: bool,
}
impl TypedGenerator for EventBlobsGeneratorI32Test01 {
type RustScalar = i32;
}
impl EventBlobsGeneratorI32Test01 {
pub fn new(node_ix: u64, node_count: u64, range: SeriesRange) -> Self {
let range = match range {
SeriesRange::TimeRange(k) => k,
SeriesRange::PulseRange(_) => todo!(),
};
let dts = MS * 500 * node_count as u64;
let ts = (range.beg / dts + node_ix) * dts;
let tsend = range.end;
Self {
ts,
dts,
tsend,
c1: 0,
scalar_type: ScalarType::I32,
be: true,
shape: Shape::Scalar,
timeout: None,
done: false,
done_range_final: false,
}
}
fn make_batch(&mut self) -> Sitemty<EventFull> {
type T = i32;
let mut item = EventFull::empty();
let mut ts = self.ts;
loop {
if self.ts >= self.tsend || item.byte_estimate() > 400 {
break;
}
let pulse = ts;
let value = (ts / self.dts) as T;
item.add_event(
ts,
pulse,
Some(value.to_be_bytes().to_vec()),
None,
self.scalar_type.clone(),
self.be,
self.shape.clone(),
None,
);
ts += self.dts;
}
self.ts = ts;
let w = sitem_data(item);
w
}
}
impl Stream for EventBlobsGeneratorI32Test01 {
type Item = Sitemty<EventFull>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.done_range_final {
Ready(None)
} else if self.ts >= self.tsend {
self.done = true;
self.done_range_final = true;
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))))
} else if false {
// To use the generator without throttling, use this scope
Ready(Some(self.make_batch()))
} else if let Some(fut) = self.timeout.as_mut() {
match fut.poll_unpin(cx) {
Ready(()) => {
self.timeout = None;
Ready(Some(self.make_batch()))
}
Pending => Pending,
}
} else {
self.timeout = Some(Box::pin(tokio::time::sleep(Duration::from_millis(2))));
continue;
};
}
}
}