daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP refactor channel config usage

Browse Source
This commit is contained in:
Dominik Werder
2023-06-19 10:04:56 +02:00
parent 44dd43240b
commit 0daa66a800
12 changed files with 303 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodenet/src/channelconfig.rs
+102 -73
View File
@@ -1,99 +1,128 @@
use err::Error;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::ChConf;
use netpod::timeunits::DAY;
use netpod::ByteOrder;
use netpod::ChannelTypeConfigGen;
use netpod::NodeConfigCached;
use netpod::ScalarType;
use netpod::SfChFetchInfo;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsNano;
const TEST_BACKEND: &str = "testbackend-00";
pub async fn channel_config(range: NanoRange, channel: SfDbChannel, ncc: &NodeConfigCached) -> Result<ChConf, Error> {
if channel.backend() == TEST_BACKEND {
let backend = channel.backend().into();
// TODO the series-ids here are just random. Need to integrate with better test setup.
let ret = if channel.name() == "scalar-i32-be" {
let ret = ChConf {
backend,
series: Some(2),
name: channel.name().into(),
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else if channel.name() == "wave-f64-be-n21" {
let ret = ChConf {
backend,
series: Some(3),
name: channel.name().into(),
scalar_type: ScalarType::F64,
shape: Shape::Wave(21),
};
Ok(ret)
} else if channel.name() == "const-regular-scalar-i32-be" {
let ret = ChConf {
backend,
series: Some(4),
name: channel.name().into(),
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else if channel.name() == "test-gen-i32-dim0-v00" {
let ret = ChConf {
backend,
series: Some(5),
name: channel.name().into(),
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else if channel.name() == "test-gen-i32-dim0-v01" {
let ret = ChConf {
backend,
series: Some(6),
name: channel.name().into(),
scalar_type: ScalarType::I32,
shape: Shape::Scalar,
};
Ok(ret)
} else if channel.name() == "test-gen-f64-dim1-v00" {
let ret = ChConf {
backend,
series: Some(7),
name: channel.name().into(),
scalar_type: ScalarType::F64,
shape: Shape::Wave(21),
};
Ok(ret)
} else {
error!("no test information");
Err(Error::with_msg_no_trace(format!("no test information"))
.add_public_msg("No channel config for test channel {:?}"))
};
fn channel_config_test_backend(channel: SfDbChannel) -> Result<ChannelTypeConfigGen, Error> {
let backend = channel.backend();
let ret = if channel.name() == "scalar-i32-be" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
2,
TsNano(DAY),
ByteOrder::Big,
ScalarType::I32,
Shape::Scalar,
);
ret
} else if channel.name() == "wave-f64-be-n21" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
3,
TsNano(DAY),
ByteOrder::Big,
ScalarType::F64,
Shape::Wave(21),
);
ret
} else if channel.name() == "const-regular-scalar-i32-be" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
2,
TsNano(DAY),
ByteOrder::Big,
ScalarType::I32,
Shape::Scalar,
);
ret
} else if channel.name() == "test-gen-i32-dim0-v00" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
2,
TsNano(DAY),
ByteOrder::Big,
ScalarType::I32,
Shape::Scalar,
);
ret
} else if channel.name() == "test-gen-i32-dim0-v01" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
2,
TsNano(DAY),
ByteOrder::Big,
ScalarType::I32,
Shape::Scalar,
);
ret
} else if channel.name() == "test-gen-f64-dim1-v00" {
let ret = SfChFetchInfo::new(
backend,
channel.name(),
3,
TsNano(DAY),
ByteOrder::Big,
ScalarType::F64,
Shape::Wave(21),
);
ret
} else {
error!("no test information");
return Err(Error::with_msg_no_trace(format!("no test information"))
.add_public_msg("No channel config for test channel {:?}"));
};
Ok(ChannelTypeConfigGen::SfDatabuffer(ret))
}
pub async fn channel_config(
range: NanoRange,
channel: SfDbChannel,
ncc: &NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
if channel.backend() == TEST_BACKEND {
channel_config_test_backend(channel)
} else if ncc.node_config.cluster.scylla.is_some() {
debug!("try to get ChConf for scylla type backend");
let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
.await
.map_err(Error::from)?;
Ok(ret)
Ok(ChannelTypeConfigGen::Scylla(ret))
} else if ncc.node.sf_databuffer.is_some() {
debug!("channel_config channel {channel:?}");
let config = disk::channelconfig::channel_config_best_match(range, channel.clone(), ncc)
.await?
.ok_or_else(|| Error::with_msg_no_trace("config entry not found"))?;
debug!("channel_config config {config:?}");
let ret = ChConf {
backend: config.channel.backend().into(),
series: channel.series(),
name: config.channel.name().into(),
scalar_type: config.scalar_type,
shape: config.shape,
};
let ret = SfChFetchInfo::new(
config.channel.backend(),
config.channel.name(),
config.keyspace,
config.time_bin_size,
config.byte_order,
config.scalar_type,
config.shape,
);
let ret = ChannelTypeConfigGen::SfDatabuffer(ret);
Ok(ret)
} else {
err::todoval()
return Err(
Error::with_msg_no_trace(format!("no channel config for backend {}", channel.backend()))
.add_public_msg(format!("no channel config for backend {}", channel.backend())),
);
}
}
nodenet/src/configquorum.rs
+4 -4
View File
@@ -5,7 +5,7 @@ use netpod::NodeConfigCached;
use netpod::SfChFetchInfo;
use netpod::SfDbChannel;
async fn find_sf_channel_config_basics_quorum() -> Result<SfChFetchInfo, Error> {
async fn find_sf_ch_config_quorum() -> Result<SfChFetchInfo, Error> {
type _A = SfDbChannel;
type _B = SfDbChConf;
// TODO create new endpoint which only returns the most matching config entry
@@ -17,10 +17,10 @@ pub async fn find_config_basics_quorum(
channel: &SfDbChannel,
ncc: &NodeConfigCached,
) -> Result<ChannelTypeConfigGen, Error> {
if let Some(cfg) = &ncc.node.sf_databuffer {
let ret: SfChFetchInfo = err::todoval();
if let Some(_cfg) = &ncc.node.sf_databuffer {
let ret: SfChFetchInfo = find_sf_ch_config_quorum().await?;
Ok(ChannelTypeConfigGen::SfDatabuffer(ret))
} else if let Some(cfg) = &ncc.node_config.cluster.scylla {
} else if let Some(_cfg) = &ncc.node_config.cluster.scylla {
let ret = dbconn::channelconfig::chconf_from_scylla_type_backend(&channel, ncc)
.await
.map_err(Error::from)?;
nodenet/src/scylla.rs
+6 -12
View File
@@ -1,12 +1,9 @@
use err::anyhow::Context;
use err::Error;
use futures_util::Stream;
use futures_util::StreamExt;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::Appendable;
use items_0::Empty;
use items_2::channelevents::ChannelEvents;
use netpod::log::*;
use netpod::ChConf;
@@ -19,19 +16,16 @@ pub async fn scylla_channel_event_stream(
evq: PlainEventsQuery,
chconf: ChConf,
scyco: &ScyllaConfig,
node_config: &NodeConfigCached,
_ncc: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
// TODO depends in general on the query
// TODO why both in PlainEventsQuery and as separate parameter? Check other usages.
let do_one_before_range = false;
// TODO use better builder pattern with shortcuts for production and dev defaults
let f = crate::channelconfig::channel_config(evq.range().try_into()?, evq.channel().clone(), node_config)
.await
.map_err(|e| Error::with_msg_no_trace(format!("{e:?}")))?;
let scy = scyllaconn::create_scy_session(scyco).await?;
let series = f.try_series().context("scylla_channel_event_stream")?;
let scalar_type = f.scalar_type;
let shape = f.shape;
let series = chconf.series();
let scalar_type = chconf.scalar_type();
let shape = chconf.shape();
let do_test_stream_error = false;
let with_values = evq.need_value_data();
debug!("Make EventsStreamScylla for {series:?} {scalar_type:?} {shape:?}");
@@ -39,8 +33,8 @@ pub async fn scylla_channel_event_stream(
series,
evq.range().into(),
do_one_before_range,
scalar_type,
shape,
scalar_type.clone(),
shape.clone(),
with_values,
scy,
do_test_stream_error,