daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify channel config lookup

Browse Source
This commit is contained in:
Dominik Werder
2024-07-24 19:45:30 +02:00
parent 3889d8bf37
commit 8f383050f5
19 changed files with 152 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/dbconn/src/channelconfig.rs
+20 -10
View File
@@ -6,6 +6,8 @@ use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::ChConf;
use netpod::ScalarType;
use netpod::SeriesKind;
use netpod::SfDbChannel;
use netpod::Shape;
use netpod::TsMs;
use std::time::Duration;
@@ -20,12 +22,11 @@ use tokio_postgres::Client;
/// In the future, we can even try to involve time range information for that, but backends like
/// old archivers and sf databuffer do not support such lookup.
pub(super) async fn chconf_best_matching_for_name_and_range(
backend: &str,
name: &str,
channel: SfDbChannel,
range: NanoRange,
pg: &Client,
) -> Result<ChConf, Error> {
debug!("chconf_best_matching_for_name_and_range {backend} {name} {range:?}");
debug!("chconf_best_matching_for_name_and_range {channel:?} {range:?}");
#[cfg(DISABLED)]
if ncc.node_config.cluster.scylla.is_none() {
let e = Error::with_msg_no_trace(format!(
@@ -44,12 +45,17 @@ pub(super) async fn chconf_best_matching_for_name_and_range(
let sql = concat!(
"select unnest(tscs) as tsc, series, scalar_type, shape_dims",
" from series_by_channel",
" where kind = 2 and facility = $1 and channel = $2",
" where facility = $1",
" and channel = $2",
" and kind = $3",
" order by tsc",
);
let res = pg.query(sql, &[&backend, &name]).await.err_conv()?;
let res = pg
.query(sql, &[&channel.backend(), &channel.name(), &channel.kind().to_db_i16()])
.await
.err_conv()?;
if res.len() == 0 {
let e = Error::with_public_msg_no_trace(format!("can not find channel information for {name}"));
let e = Error::with_public_msg_no_trace(format!("can not find channel information for {channel:?} {range:?}"));
warn!("{e}");
Err(e)
} else if res.len() > 1 {
@@ -64,12 +70,13 @@ pub(super) async fn chconf_best_matching_for_name_and_range(
let _scalar_type = ScalarType::from_scylla_i32(scalar_type)?;
let _shape = Shape::from_scylla_shape_dims(&shape_dims)?;
let tsms = tsc.signed_duration_since(DateTime::UNIX_EPOCH).num_milliseconds() as u64;
let ts = TsMs(tsms);
let ts = TsMs::from_ms_u64(tsms);
rows.push((ts, series));
}
let tsmss: Vec<_> = rows.iter().map(|x| x.0.clone()).collect();
let range = (TsMs(range.beg / 1000), TsMs(range.end / 1000));
let res = decide_best_matching_index(range, &tsmss)?;
let backend = channel.backend().into();
let ch_conf = chconf_for_series(backend, rows[res].1, pg).await?;
Ok(ch_conf)
} else {
@@ -80,9 +87,10 @@ pub(super) async fn chconf_best_matching_for_name_and_range(
// TODO can I get a slice from psql driver?
let shape_dims: Vec<i32> = r.get(3);
let series = series as u64;
let kind = channel.kind();
let scalar_type = ScalarType::from_scylla_i32(scalar_type)?;
let shape = Shape::from_scylla_shape_dims(&shape_dims)?;
let ret = ChConf::new(backend, series, scalar_type, shape, name);
let ret = ChConf::new(channel.backend(), series, kind, scalar_type, shape, channel.name());
Ok(ret)
}
}
@@ -194,7 +202,7 @@ fn test_decide_best_matching_index_after_01() {
pub(super) async fn chconf_for_series(backend: &str, series: u64, pg: &Client) -> Result<ChConf, Error> {
let res = pg
.query(
"select channel, scalar_type, shape_dims from series_by_channel where facility = $1 and series = $2",
"select channel, scalar_type, shape_dims, kind from series_by_channel where facility = $1 and series = $2",
&[&backend, &(series as i64)],
)
.await
@@ -211,7 +219,9 @@ pub(super) async fn chconf_for_series(backend: &str, series: u64, pg: &Client) -
let scalar_type = ScalarType::from_dtype_index(row.get::<_, i32>(1) as u8)?;
// TODO can I get a slice from psql driver?
let shape = Shape::from_scylla_shape_dims(&row.get::<_, Vec<i32>>(2))?;
let ret = ChConf::new(backend, series, scalar_type, shape, name);
let kind: i16 = row.get(3);
let kind = SeriesKind::from_db_i16(kind)?;
let ret = ChConf::new(backend, series, kind, scalar_type, shape, name);
Ok(ret)
}
}
crates/dbconn/src/dbconn.rs
-1
View File
@@ -1,6 +1,5 @@
pub mod channelconfig;
pub mod channelinfo;
pub mod query;
pub mod scan;
pub mod search;
pub mod worker;
crates/dbconn/src/query.rs
-48
View File
@@ -1,48 +0,0 @@
use crate::create_connection;
use crate::ErrConv;
use err::Error;
use netpod::log::*;
use netpod::NodeConfigCached;
use netpod::SfDbChannel;
// For sf-databuffer backend, given a Channel, try to complete the information if only id is given.
async fn sf_databuffer_fetch_channel_by_series(
channel: SfDbChannel,
ncc: &NodeConfigCached,
) -> Result<SfDbChannel, Error> {
let me = "sf_databuffer_fetch_channel_by_series";
info!("{me}");
// TODO should not be needed at some point.
if channel.backend().is_empty() || channel.name().is_empty() {
if let Some(series) = channel.series() {
if series < 1 {
error!("{me} bad input: {channel:?}");
Err(Error::with_msg_no_trace(format!("{me} bad input: {channel:?}")))
} else {
info!("{me} do the lookup");
let series = channel
.series()
.ok_or_else(|| Error::with_msg_no_trace("no series id given"))? as i64;
let (pgcon, _pgjh) = create_connection(&ncc.node_config.cluster.database).await?;
let mut rows = pgcon
.query("select name from channels where rowid = $1", &[&series])
.await
.err_conv()?;
if let Some(row) = rows.pop() {
info!("{me} got a row {row:?}");
let name: String = row.get(0);
let channel = SfDbChannel::from_full(&ncc.node_config.cluster.backend, channel.series(), name);
info!("{me} return {channel:?}");
Ok(channel)
} else {
info!("{me} nothing found");
Err(Error::with_msg_no_trace("can not find series"))
}
}
} else {
Err(Error::with_msg_no_trace(format!("{me} bad input: {channel:?}")))
}
} else {
Ok(channel)
}
}
crates/dbconn/src/worker.rs
+7 -8
View File
@@ -10,6 +10,7 @@ use netpod::ChConf;
use netpod::ChannelSearchQuery;
use netpod::ChannelSearchResult;
use netpod::Database;
use netpod::SeriesKind;
use netpod::SfDbChannel;
use taskrun::tokio;
use tokio::task::JoinHandle;
@@ -38,7 +39,7 @@ impl err::ToErr for Error {
#[derive(Debug)]
enum Job {
ChConfBestMatchingNameRange(String, String, NanoRange, Sender<Result<ChConf, Error>>),
ChConfBestMatchingNameRange(SfDbChannel, NanoRange, Sender<Result<ChConf, Error>>),
ChConfForSeries(String, u64, Sender<Result<ChConf, Error>>),
InfoForSeriesIds(
Vec<u64>,
@@ -70,12 +71,11 @@ impl PgQueue {
pub async fn chconf_best_matching_name_range(
&self,
backend: &str,
name: &str,
channel: SfDbChannel,
range: NanoRange,
) -> Result<Receiver<Result<ChConf, Error>>, Error> {
let (tx, rx) = async_channel::bounded(1);
let job = Job::ChConfBestMatchingNameRange(backend.into(), name.into(), range, tx);
let job = Job::ChConfBestMatchingNameRange(channel, range, tx);
self.tx.send(job).await.map_err(|_| Error::ChannelSend)?;
Ok(rx)
}
@@ -144,10 +144,9 @@ impl PgWorker {
}
};
match job {
Job::ChConfBestMatchingNameRange(backend, name, range, tx) => {
Job::ChConfBestMatchingNameRange(channel, range, tx) => {
let res =
crate::channelconfig::chconf_best_matching_for_name_and_range(&backend, &name, range, &self.pg)
.await;
crate::channelconfig::chconf_best_matching_for_name_and_range(channel, range, &self.pg).await;
if tx.send(res.map_err(Into::into)).await.is_err() {
// TODO count for stats
}
@@ -223,7 +222,7 @@ async fn find_sf_channel_by_series(
}
if let Some(row) = rows.into_iter().next() {
let name = row.get::<_, String>(0);
let channel = SfDbChannel::from_full(channel.backend(), channel.series(), name);
let channel = SfDbChannel::from_full(channel.backend(), channel.series(), name, SeriesKind::default());
Ok(channel)
} else {
return Err(FindChannelError::NoFound);