daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor config entry matching

Browse Source
This commit is contained in:
Dominik Werder
2023-06-14 14:21:28 +02:00
parent 4899d71022
commit c60243c646
15 changed files with 249 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
disk/src/cache.rs
View File

@@ -20,8 +20,8 @@ use tiny_keccak::Hasher;
// No longer needed for scylla-based caching.
pub fn node_ix_for_patch(patch_coord: &PreBinnedPatchCoordEnum, channel: &SfDbChannel, cluster: &Cluster) -> u32 {
let mut hash = tiny_keccak::Sha3::v256();
hash.update(channel.backend.as_bytes());
hash.update(channel.name.as_bytes());
hash.update(channel.backend().as_bytes());
hash.update(channel.name().as_bytes());
/*hash.update(&patch_coord.patch_beg().to_le_bytes());
hash.update(&patch_coord.patch_end().to_le_bytes());
hash.update(&patch_coord.bin_t_len().to_le_bytes());
@@ -53,8 +53,8 @@ impl CacheFileDesc {
pub fn hash(&self) -> String {
let mut h = tiny_keccak::Sha3::v256();
h.update(b"V000");
h.update(self.channel.backend.as_bytes());
h.update(self.channel.name.as_bytes());
h.update(self.channel.backend().as_bytes());
h.update(self.channel.name().as_bytes());
h.update(format!("{}", self.agg_kind).as_bytes());
//h.update(&self.patch.spec().bin_t_len().to_le_bytes());
//h.update(&self.patch.spec().patch_t_len().to_le_bytes());
@@ -67,8 +67,8 @@ impl CacheFileDesc {
pub fn hash_channel(&self) -> String {
let mut h = tiny_keccak::Sha3::v256();
h.update(b"V000");
h.update(self.channel.backend.as_bytes());
h.update(self.channel.name.as_bytes());
h.update(self.channel.backend().as_bytes());
h.update(self.channel.name().as_bytes());
let mut buf = [0; 32];
h.finalize(&mut buf);
hex::encode(&buf)
@@ -83,7 +83,7 @@ impl CacheFileDesc {
.join("cache")
.join(&hc[0..3])
.join(&hc[3..6])
.join(&self.channel.name)
.join(self.channel.name())
.join(format!("{}", self.agg_kind))
/*.join(format!(
"{:010}-{:010}",

79
disk/src/channelconfig.rs
View File

@@ -6,50 +6,57 @@ use netpod::SfDbChannel;
use parse::channelconfig::extract_matching_config_entry;
use parse::channelconfig::read_local_config;
use parse::channelconfig::ChannelConfigs;
use parse::channelconfig::MatchingConfigEntry;
use parse::channelconfig::ConfigEntry;
pub async fn config(
range: NanoRange,
pub async fn config_entry_best_match(
range: &NanoRange,
channel: SfDbChannel,
node_config: &NodeConfigCached,
) -> Result<SfDbChConf, Error> {
let channel_configs = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(&range, &channel_configs) {
) -> Result<Option<ConfigEntry>, Error> {
let channel_config = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(range, &channel_config) {
Ok(k) => k,
Err(e) => return Err(e)?,
};
let entry = match entry_res {
MatchingConfigEntry::None => {
return Err(Error::with_public_msg(format!(
"disk::channelconfig no config entry found for {:?}",
channel
)))?
}
MatchingConfigEntry::Multiple => {
return Err(Error::with_public_msg(format!(
"disk::channelconfig multiple config entries in range found for {:?}",
channel
)))?
}
MatchingConfigEntry::Entry(entry) => entry,
};
let shape = match entry.to_shape() {
Ok(k) => k,
Err(e) => return Err(e)?,
};
let channel_config = SfDbChConf {
channel: channel.clone(),
keyspace: entry.ks as u8,
time_bin_size: entry.bs.clone(),
shape,
scalar_type: entry.scalar_type.clone(),
byte_order: entry.byte_order.clone(),
array: entry.is_array,
compression: entry.is_compressed,
};
Ok(channel_config)
match entry_res.best() {
None => Ok(None),
Some(x) => Ok(Some(x.clone())),
}
}
pub async fn configs(channel: SfDbChannel, node_config: &NodeConfigCached) -> Result<ChannelConfigs, Error> {
read_local_config(channel.clone(), node_config.clone()).await
}
pub async fn channel_config_best_match(
range: NanoRange,
channel: SfDbChannel,
node_config: &NodeConfigCached,
) -> Result<Option<SfDbChConf>, Error> {
let best = config_entry_best_match(&range, channel.clone(), node_config).await?;
let channel_configs = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(&range, &channel_configs) {
Ok(k) => k,
Err(e) => return Err(e),
};
match entry_res.best() {
None => Ok(None),
Some(entry) => {
let shape = match entry.to_shape() {
Ok(k) => k,
Err(e) => return Err(e)?,
};
let channel_config = SfDbChConf {
channel: channel.clone(),
keyspace: entry.ks as u8,
time_bin_size: entry.bs.clone(),
shape,
scalar_type: entry.scalar_type.clone(),
byte_order: entry.byte_order.clone(),
array: entry.is_array,
compression: entry.is_compressed,
};
Ok(Some(channel_config))
}
}
}

8
disk/src/gen.rs
View File

@@ -152,12 +152,12 @@ async fn gen_node(split: u32, node: &Node, ensemble: &Ensemble) -> Result<(), Er
async fn gen_channel(chn: &ChannelGenProps, split: u32, node: &Node, ensemble: &Ensemble) -> Result<(), Error> {
let sfc = node.sf_databuffer.as_ref().unwrap();
let config_path = sfc.data_base_path.join("config").join(&chn.config.channel.name);
let config_path = sfc.data_base_path.join("config").join(chn.config.channel.name());
let channel_path = sfc
.data_base_path
.join(format!("{}_{}", sfc.ksprefix, chn.config.keyspace))
.join("byTime")
.join(&chn.config.channel.name);
.join(chn.config.channel.name());
tokio::fs::create_dir_all(&channel_path).await?;
gen_config(&config_path, &chn.config, node, ensemble)
.await
@@ -200,7 +200,7 @@ async fn gen_config(config_path: &Path, config: &SfDbChConf, _node: &Node, _ense
let mut buf = BytesMut::with_capacity(1024 * 1);
let ver = 0;
buf.put_i16(ver);
let cnenc = config.channel.name.as_bytes();
let cnenc = config.channel.name().as_bytes();
let len1 = cnenc.len() + 8;
buf.put_i32(len1 as i32);
buf.put(cnenc);
@@ -385,7 +385,7 @@ async fn gen_timebin(
async fn gen_datafile_header(file: &mut CountedFile, config: &SfDbChConf) -> Result<(), Error> {
let mut buf = BytesMut::with_capacity(1024);
let cnenc = config.channel.name.as_bytes();
let cnenc = config.channel.name().as_bytes();
let len1 = cnenc.len() + 8;
buf.put_i16(0);
buf.put_i32(len1 as i32);

8
disk/src/paths.rs
View File

@@ -18,7 +18,7 @@ pub fn datapath(timebin: u64, config: &SfDbChConf, split: u32, node: &Node) -> P
config.keyspace
))
.join("byTime")
.join(config.channel.name.clone())
.join(config.channel.name())
.join(format!("{:019}", timebin))
.join(format!("{:010}", split))
.join(format!("{:019}_00000_Data", config.time_bin_size.ns() / MS))
@@ -36,7 +36,7 @@ pub async fn datapaths_for_timebin(timebin: u64, config: &SfDbChConf, node: &Nod
.data_base_path
.join(format!("{}_{}", sfc.ksprefix, config.keyspace))
.join("byTime")
.join(config.channel.name.clone())
.join(config.channel.name())
.join(format!("{:019}", timebin));
let rd = tokio::fs::read_dir(timebin_path).await?;
let mut rd = tokio_stream::wrappers::ReadDirStream::new(rd);
@@ -71,7 +71,7 @@ pub async fn datapaths_for_timebin(timebin: u64, config: &SfDbChConf, node: &Nod
.data_base_path
.join(format!("{}_{}", sfc.ksprefix, config.keyspace))
.join("byTime")
.join(config.channel.name.clone())
.join(config.channel.name())
.join(format!("{:019}", timebin))
.join(format!("{:010}", split))
.join(format!("{:019}_00000_Data", config.time_bin_size.ns() / MS));
@@ -86,7 +86,7 @@ pub fn channel_timebins_dir_path(channel_config: &SfDbChConf, node: &Node) -> Re
.data_base_path
.join(format!("{}_{}", sfc.ksprefix, channel_config.keyspace))
.join("byTime")
.join(&channel_config.channel.name);
.join(channel_config.channel.name());
Ok(ret)
}

83
disk/src/raw/conn.rs
View File

@@ -1,3 +1,4 @@
use crate::channelconfig::config_entry_best_match;
use crate::eventblobs::EventChunkerMultifile;
use crate::eventchunker::EventChunkerConf;
use crate::raw::generated::EventBlobsGeneratorI32Test00;
@@ -20,10 +21,7 @@ use netpod::ChConf;
use netpod::DiskIoTune;
use netpod::NodeConfigCached;
use netpod::SfDbChannel;
use parse::channelconfig::extract_matching_config_entry;
use parse::channelconfig::read_local_config;
use parse::channelconfig::ConfigEntry;
use parse::channelconfig::MatchingConfigEntry;
use query::api4::events::PlainEventsQuery;
use std::pin::Pin;
@@ -63,32 +61,16 @@ pub async fn make_event_pipe(
chconf: ChConf,
ncc: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
info!("---------- disk::raw::conn::make_event_pipe");
if false {
match dbconn::channel_exists(&evq.channel(), &ncc).await {
Ok(_) => (),
Err(e) => return Err(e)?,
}
}
let chn = evq.channel().clone();
let chn = if chn.name().is_empty() {
if let Some(series) = chn.series() {
if series < 1 {
error!("BAD QUERY: {evq:?}");
return Err(Error::with_msg_no_trace(format!("BAD QUERY: {evq:?}")));
} else {
dbconn::query::sf_databuffer_fetch_channel_by_series(chn, ncc).await?
}
} else {
chn
}
} else {
chn
};
// sf-databuffer type backends identify channels by their (backend, name) only.
let channel = evq.channel().clone();
let range = evq.range().clone();
let channel_config = crate::channelconfig::config(evq.range().try_into()?, chn, ncc).await;
let channel_config = match channel_config {
Ok(x) => x,
let x = crate::channelconfig::channel_config_best_match(evq.range().try_into()?, channel, ncc).await;
let channel_config = match x {
Ok(Some(x)) => x,
Ok(None) => {
error!("make_event_pipe can not find config");
return Err(Error::with_msg_no_trace("make_event_pipe can not find config"));
}
Err(e) => {
error!("make_event_pipe can not find config");
if e.msg().contains("ErrorKind::NotFound") {
@@ -96,7 +78,7 @@ pub async fn make_event_pipe(
let s = futures_util::stream::empty();
return Ok(Box::pin(s));
} else {
return Err(e)?;
return Err(e);
}
}
};
@@ -128,35 +110,6 @@ pub async fn make_event_pipe(
Ok(pipe)
}
pub async fn get_applicable_entry(
range: &NanoRange,
channel: SfDbChannel,
node_config: &NodeConfigCached,
) -> Result<ConfigEntry, Error> {
debug!("disk::raw::conn::get_applicable_entry");
let channel_config = read_local_config(channel.clone(), node_config.clone()).await?;
let entry_res = match extract_matching_config_entry(range, &channel_config) {
Ok(k) => k,
Err(e) => return Err(e)?,
};
let entry = match entry_res {
MatchingConfigEntry::None => {
return Err(Error::with_public_msg(format!(
"get_applicable_entry no config entry found {:?}",
channel
)))?
}
MatchingConfigEntry::Multiple => {
return Err(Error::with_public_msg(format!(
"get_applicable_entry multiple config entries found for {:?}",
channel
)))?
}
MatchingConfigEntry::Entry(entry) => entry,
};
Ok(entry.clone())
}
pub fn make_local_event_blobs_stream(
range: NanoRange,
channel: SfDbChannel,
@@ -264,8 +217,13 @@ pub async fn make_event_blobs_pipe_real(
}
let expand = evq.one_before_range();
let range = evq.range();
let entry = match get_applicable_entry(&evq.range().try_into()?, evq.channel().clone(), node_config).await {
Ok(x) => x,
let entry = match config_entry_best_match(&evq.range().try_into()?, evq.channel().clone(), node_config).await {
Ok(Some(x)) => x,
Ok(None) => {
let e = Error::with_msg_no_trace("no config entry found");
error!("{e}");
return Err(e);
}
Err(e) => {
if e.to_public_error().msg().contains("no config entry found") {
let item = items_0::streamitem::LogItem {
@@ -280,7 +238,6 @@ pub async fn make_event_blobs_pipe_real(
}
};
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
type ItemType = Sitemty<EventFull>;
// TODO should depend on host config
let do_local = node_config.node_config.cluster.is_central_storage;
let pipe = if do_local {
@@ -306,7 +263,9 @@ pub async fn make_event_blobs_pipe_real(
DiskIoTune::default(),
node_config,
)?;
/*let s = event_blobs.map(|item: ItemType| Box::new(item) as Box<dyn Framable + Send>);
/*
type ItemType = Sitemty<EventFull>;
let s = event_blobs.map(|item: ItemType| Box::new(item) as Box<dyn Framable + Send>);
//let s = tracing_futures::Instrumented::instrument(s, tracing::info_span!("make_event_blobs_pipe"));
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
pipe = Box::pin(s);