daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename SfDbChannel

Browse Source
This commit is contained in:
Dominik Werder
2023-06-13 16:17:08 +02:00
parent 9c0062d27c
commit 7c77b07db5
40 changed files with 733 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
disk/src/aggtest.rs
View File

@@ -2,16 +2,17 @@ use crate::eventblobs::EventChunkerMultifile;
use crate::eventchunker::EventChunkerConf;
use crate::AggQuerySingleChannel;
use crate::SfDbChConf;
use err::Error;
use netpod::range::evrange::NanoRange;
use netpod::test_data_base_path_databuffer;
use netpod::timeunits::*;
use netpod::ByteOrder;
use netpod::ByteSize;
use netpod::Channel;
use netpod::DiskIoTune;
use netpod::Node;
use netpod::ScalarType;
use netpod::SfDatabuffer;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
@@ -38,7 +39,7 @@ pub fn make_test_node(id: u32) -> Node {
fn agg_x_dim_0() {
taskrun::run(async {
agg_x_dim_0_inner().await;
Ok(())
Ok::<_, Error>(())
})
.unwrap();
}
@@ -47,7 +48,7 @@ async fn agg_x_dim_0_inner() {
let node = make_test_node(0);
let query = AggQuerySingleChannel {
channel_config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: "sf-databuffer".into(),
name: "S10BC01-DBAM070:EOM1_T1".into(),
series: None,
@@ -92,7 +93,7 @@ async fn agg_x_dim_0_inner() {
fn agg_x_dim_1() {
taskrun::run(async {
agg_x_dim_1_inner().await;
Ok(())
Ok::<_, Error>(())
})
.unwrap();
}
@@ -104,7 +105,7 @@ async fn agg_x_dim_1_inner() {
let node = make_test_node(0);
let query = AggQuerySingleChannel {
channel_config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: "ks".into(),
name: "wave1".into(),
series: None,

10
disk/src/cache.rs
View File

@@ -3,10 +3,10 @@ use err::Error;
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::AggKind;
use netpod::Channel;
use netpod::Cluster;
use netpod::NodeConfigCached;
use netpod::PreBinnedPatchCoordEnum;
use netpod::SfDbChannel;
use serde::Deserialize;
use serde::Serialize;
use std::collections::VecDeque;
@@ -18,7 +18,7 @@ use tiny_keccak::Hasher;
// For file-based caching, this determined the node where the cache file is located.
// No longer needed for scylla-based caching.
pub fn node_ix_for_patch(patch_coord: &PreBinnedPatchCoordEnum, channel: &Channel, cluster: &Cluster) -> u32 {
pub fn node_ix_for_patch(patch_coord: &PreBinnedPatchCoordEnum, channel: &SfDbChannel, cluster: &Cluster) -> u32 {
let mut hash = tiny_keccak::Sha3::v256();
hash.update(channel.backend.as_bytes());
hash.update(channel.name.as_bytes());
@@ -36,13 +36,13 @@ pub fn node_ix_for_patch(patch_coord: &PreBinnedPatchCoordEnum, channel: &Channe
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CacheFileDesc {
// What identifies a cached file?
channel: Channel,
channel: SfDbChannel,
patch: PreBinnedPatchCoordEnum,
agg_kind: AggKind,
}
impl CacheFileDesc {
pub fn new(channel: Channel, patch: PreBinnedPatchCoordEnum, agg_kind: AggKind) -> Self {
pub fn new(channel: SfDbChannel, patch: PreBinnedPatchCoordEnum, agg_kind: AggKind) -> Self {
Self {
channel,
patch,
@@ -104,7 +104,7 @@ pub async fn write_pb_cache_min_max_avg_scalar<T>(
values: T,
patch: PreBinnedPatchCoordEnum,
agg_kind: AggKind,
channel: Channel,
channel: SfDbChannel,
node_config: NodeConfigCached,
) -> Result<WrittenPbCache, Error>
where

10
disk/src/channelconfig.rs
View File

@@ -1,14 +1,18 @@
use crate::SfDbChConf;
use err::Error;
use netpod::range::evrange::NanoRange;
use netpod::Channel;
use netpod::NodeConfigCached;
use netpod::SfDbChannel;
use parse::channelconfig::extract_matching_config_entry;
use parse::channelconfig::read_local_config;
use parse::channelconfig::ChannelConfigs;
use parse::channelconfig::MatchingConfigEntry;
pub async fn config(range: NanoRange, channel: Channel, node_config: &NodeConfigCached) -> Result<SfDbChConf, Error> {
pub async fn config(
range: NanoRange,
channel: SfDbChannel,
node_config: &NodeConfigCached,
) -> Result<SfDbChConf, Error> {
let channel_configs = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(&range, &channel_configs) {
Ok(k) => k,
@@ -46,6 +50,6 @@ pub async fn config(range: NanoRange, channel: Channel, node_config: &NodeConfig
Ok(channel_config)
}
pub async fn configs(channel: Channel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
pub async fn configs(channel: SfDbChannel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
read_local_config(channel.clone(), node_config.clone()).await
}

49
disk/src/dataopen.rs
View File

@@ -1,6 +1,5 @@
use crate::SfDbChConf;
use super::paths;
use crate::SfDbChConf;
use bytes::BytesMut;
use err::ErrStr;
use err::Error;
@@ -445,7 +444,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 23);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -463,7 +462,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -482,7 +481,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 23);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -501,7 +500,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 179);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -520,7 +519,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -539,7 +538,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -557,7 +556,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -576,7 +575,7 @@ mod test {
assert_eq!(res.file.index, true);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 184);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -594,7 +593,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, true);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -612,7 +611,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, true);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -631,7 +630,7 @@ mod test {
assert_eq!(res.file.index, true);
assert_eq!(res.file.positioned, false);
assert_eq!(res.file.pos, 0);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -655,10 +654,10 @@ mod test {
.await?;
assert_eq!(res.1, true);
assert_eq!(res.3, 75);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
Ok::<_, Error>(())
}
#[test]
@@ -673,7 +672,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -692,7 +691,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 23);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -711,7 +710,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 75);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -731,7 +730,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 2995171);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -750,7 +749,7 @@ mod test {
assert_eq!(res.found, false);
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, false);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -770,7 +769,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 23);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -790,7 +789,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 75);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -810,7 +809,7 @@ mod test {
assert_eq!(res.file.index, false);
assert_eq!(res.file.positioned, true);
assert_eq!(res.file.pos, 127);
Ok(())
Ok::<_, Error>(())
};
taskrun::run(fut)?;
Ok(())
@@ -822,7 +821,7 @@ mod test {
beg: DAY + HOUR * 5,
end: DAY + HOUR * 8,
};
let chn = netpod::Channel {
let chn = netpod::SfDbChannel {
backend: BACKEND.into(),
name: "scalar-i32-be".into(),
series: None,
@@ -860,7 +859,7 @@ mod test {
n = paths.len()
)));
}
Ok(())
Ok::<_, Error>(())
};
taskrun::run(task).unwrap();
}

4
disk/src/disk.rs
View File

@@ -27,11 +27,11 @@ use futures_util::StreamExt;
use futures_util::TryFutureExt;
use netpod::log::*;
use netpod::ByteOrder;
use netpod::Channel;
use netpod::DiskIoTune;
use netpod::Node;
use netpod::ReadSys;
use netpod::ScalarType;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
use serde::Deserialize;
@@ -63,7 +63,7 @@ use tokio::sync::mpsc;
// TODO move to databuffer-specific crate
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SfDbChConf {
pub channel: Channel,
pub channel: SfDbChannel,
pub keyspace: u8,
pub time_bin_size: TsNano,
pub scalar_type: ScalarType,

2
disk/src/eventblobs.rs
View File

@@ -298,7 +298,7 @@ mod test {
const BACKEND: &str = "testbackend-00";
fn read_expanded_for_range(range: NanoRange, nodeix: usize) -> Result<(usize, Vec<u64>), Error> {
let chn = netpod::Channel {
let chn = netpod::SfDbChannel {
backend: BACKEND.into(),
name: "scalar-i32-be".into(),
series: None,

145
disk/src/eventchunker.rs
View File

@@ -53,11 +53,15 @@ pub struct EventChunker {
seen_after_range_count: usize,
unordered_warn_count: usize,
repeated_ts_warn_count: usize,
config_mismatch_discard: usize,
}
impl Drop for EventChunker {
fn drop(&mut self) {
// TODO collect somewhere
if self.config_mismatch_discard != 0 {
warn!("config_mismatch_discard {}", self.config_mismatch_discard);
}
debug!(
"EventChunker Drop Stats:\ndecomp_dt_histo: {:?}\nitem_len_emit_histo: {:?}",
self.decomp_dt_histo, self.item_len_emit_histo
@@ -124,6 +128,7 @@ impl EventChunker {
seen_after_range_count: 0,
unordered_warn_count: 0,
repeated_ts_warn_count: 0,
config_mismatch_discard: 0,
}
}
@@ -352,73 +357,111 @@ impl EventChunker {
let type_size = scalar_type.bytes() as u32;
let ele_count = value_bytes / type_size as u64;
let ele_size = type_size;
match self.channel_config.shape {
let config_matches = match self.channel_config.shape {
Shape::Scalar => {
if is_array {
Err(Error::with_msg(format!(
"ChannelConfig expects Scalar but we find event is_array"
)))?;
if false {
error!(
"channel config mismatch {:?} {:?} {:?} {:?}",
self.channel_config, is_array, ele_count, self.dbg_path,
);
}
if false {
return Err(Error::with_msg(format!(
"ChannelConfig expects {:?} but we find event is_array",
self.channel_config,
)));
}
false
} else {
true
}
}
Shape::Wave(dim1count) => {
if dim1count != ele_count as u32 {
Err(Error::with_msg(format!(
"ChannelConfig expects {:?} but event has ele_count {}",
self.channel_config.shape, ele_count,
)))?;
if false {
error!(
"channel config mismatch {:?} {:?} {:?} {:?}",
self.channel_config, is_array, ele_count, self.dbg_path,
);
}
if false {
return Err(Error::with_msg(format!(
"ChannelConfig expects {:?} but event has ele_count {}",
self.channel_config, ele_count,
)));
}
false
} else {
true
}
}
Shape::Image(n1, n2) => {
let nt = n1 as usize * n2 as usize;
if nt != ele_count as usize {
Err(Error::with_msg(format!(
"ChannelConfig expects {:?} but event has ele_count {}",
self.channel_config.shape, ele_count,
)))?;
}
}
}
let data = &buf.as_ref()[(p1 as usize)..(p1 as usize + k1 as usize)];
let decomp = {
if self.do_decompress {
assert!(data.len() > 12);
let ts1 = Instant::now();
let decomp_bytes = (type_size * ele_count as u32) as usize;
let mut decomp = vec![0; decomp_bytes];
// TODO limit the buf slice range
match bitshuffle_decompress(
&data[12..],
&mut decomp,
ele_count as usize,
ele_size as usize,
0,
) {
Ok(c1) => {
assert!(c1 as u64 + 12 == k1);
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
// TODO analyze the histo
self.decomp_dt_histo.ingest(dt.as_secs() as u32 + dt.subsec_micros());
Some(decomp)
if false {
error!(
"channel config mismatch {:?} {:?} {:?} {:?}",
self.channel_config, is_array, ele_count, self.dbg_path,
);
}
Err(e) => {
return Err(Error::with_msg(format!("decompression failed {:?}", e)))?;
if false {
return Err(Error::with_msg(format!(
"ChannelConfig expects {:?} but event has ele_count {}",
self.channel_config, ele_count,
)));
}
false
} else {
true
}
} else {
None
}
};
ret.add_event(
ts,
pulse,
Some(data.to_vec()),
decomp,
ScalarType::from_dtype_index(type_index)?,
is_big_endian,
shape_this,
comp_this,
);
if config_matches {
let data = buf.as_ref()[(p1 as usize)..(p1 as usize + k1 as usize)].as_ref();
let decomp = {
if self.do_decompress {
assert!(data.len() > 12);
let ts1 = Instant::now();
let decomp_bytes = (type_size * ele_count as u32) as usize;
let mut decomp = vec![0; decomp_bytes];
// TODO limit the buf slice range
match bitshuffle_decompress(
&data[12..],
&mut decomp,
ele_count as usize,
ele_size as usize,
0,
) {
Ok(c1) => {
assert!(c1 as u64 + 12 == k1);
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
// TODO analyze the histo
self.decomp_dt_histo.ingest(dt.as_secs() as u32 + dt.subsec_micros());
Some(decomp)
}
Err(e) => {
return Err(Error::with_msg(format!("decompression failed {:?}", e)))?;
}
}
} else {
None
}
};
ret.add_event(
ts,
pulse,
Some(data.to_vec()),
decomp,
ScalarType::from_dtype_index(type_index)?,
is_big_endian,
shape_this,
comp_this,
);
} else {
self.config_mismatch_discard += 1;
}
} else {
if len < p1 as u32 + 4 {
let msg = format!("uncomp len: {} p1: {}", len, p1);

12
disk/src/gen.rs
View File

@@ -7,11 +7,11 @@ use err::Error;
use netpod::log::*;
use netpod::timeunits::*;
use netpod::ByteOrder;
use netpod::Channel;
use netpod::GenVar;
use netpod::Node;
use netpod::ScalarType;
use netpod::SfDatabuffer;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
use std::path::Path;
@@ -34,7 +34,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
{
let chn = ChannelGenProps {
config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: backend.clone(),
name: "scalar-i32-be".into(),
series: None,
@@ -53,7 +53,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
ensemble.channels.push(chn);
let chn = ChannelGenProps {
config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: backend.clone(),
name: "wave-f64-be-n21".into(),
series: None,
@@ -72,7 +72,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
ensemble.channels.push(chn);
let chn = ChannelGenProps {
config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: backend.clone(),
name: "wave-u16-le-n77".into(),
series: None,
@@ -91,7 +91,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
ensemble.channels.push(chn);
let chn = ChannelGenProps {
config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: backend.clone(),
name: "tw-scalar-i32-be".into(),
series: None,
@@ -110,7 +110,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
ensemble.channels.push(chn);
let chn = ChannelGenProps {
config: SfDbChConf {
channel: Channel {
channel: SfDbChannel {
backend: backend.clone(),
name: "const-regular-scalar-i32-be".into(),
series: None,

14
disk/src/raw/conn.rs
View File

@@ -17,9 +17,9 @@ use netpod::range::evrange::NanoRange;
use netpod::AggKind;
use netpod::ByteSize;
use netpod::ChConf;
use netpod::Channel;
use netpod::DiskIoTune;
use netpod::NodeConfigCached;
use netpod::SfDbChannel;
use parse::channelconfig::extract_matching_config_entry;
use parse::channelconfig::read_local_config;
use parse::channelconfig::ConfigEntry;
@@ -130,10 +130,10 @@ pub async fn make_event_pipe(
pub async fn get_applicable_entry(
range: &NanoRange,
channel: Channel,
channel: SfDbChannel,
node_config: &NodeConfigCached,
) -> Result<ConfigEntry, Error> {
info!("---------- disk::raw::conn::get_applicable_entry");
debug!("disk::raw::conn::get_applicable_entry");
let channel_config = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(range, &channel_config) {
Ok(k) => k,
@@ -159,7 +159,7 @@ pub async fn get_applicable_entry(
pub fn make_local_event_blobs_stream(
range: NanoRange,
channel: Channel,
channel: SfDbChannel,
entry: &ConfigEntry,
expand: bool,
do_decompress: bool,
@@ -208,7 +208,7 @@ pub fn make_local_event_blobs_stream(
pub fn make_remote_event_blobs_stream(
range: NanoRange,
channel: Channel,
channel: SfDbChannel,
entry: &ConfigEntry,
expand: bool,
do_decompress: bool,
@@ -216,7 +216,7 @@ pub fn make_remote_event_blobs_stream(
disk_io_tune: DiskIoTune,
node_config: &NodeConfigCached,
) -> Result<impl Stream<Item = Sitemty<EventFull>>, Error> {
info!("make_remote_event_blobs_stream");
debug!("make_remote_event_blobs_stream");
let shape = match entry.to_shape() {
Ok(k) => k,
Err(e) => return Err(e)?,
@@ -363,7 +363,7 @@ pub async fn make_event_blobs_pipe(
evq: &PlainEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventFull>> + Send>>, Error> {
info!("make_event_blobs_pipe {evq:?}");
debug!("make_event_blobs_pipe {evq:?}");
if evq.channel().backend() == TEST_BACKEND {
make_event_blobs_pipe_test(evq, node_config).await
} else {