daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor, inspect bin write index helper

Browse Source
This commit is contained in:
Dominik Werder
2025-03-21 14:45:49 +01:00
parent f8b3c1533b
commit 272cc77a32
24 changed files with 870 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/dbconn/src/search.rs
+28 -32
View File
@@ -1,9 +1,6 @@
use crate::ErrConv;
use crate::create_connection;
use crate::worker::PgQueue;
use crate::ErrConv;
use daqbuf_err as err;
use err::Error;
use netpod::log::*;
use netpod::ChannelArchiver;
use netpod::ChannelSearchQuery;
use netpod::ChannelSearchResult;
@@ -12,9 +9,25 @@ use netpod::Database;
use netpod::NodeConfigCached;
use netpod::ScalarType;
use netpod::Shape;
use netpod::log;
use serde_json::Value as JsVal;
use tokio_postgres::Client as PgClient;
macro_rules! debug { ($($arg:expr),*) => ( if true { log::debug!($($arg),*) } ); }
macro_rules! trace { ($($arg:expr),*) => ( if true { log::trace!($($arg),*) } ); }
autoerr::create_error_v1!(
name(Error, "DbconnSearch"),
enum variants {
BadShapeFromDb(String),
BadArchEngValue(String),
Pg(#[from] tokio_postgres::Error),
Other(#[from] daqbuf_err::Error),
Worker(#[from] crate::worker::Error),
},
);
pub async fn search_channel_databuffer(
query: ChannelSearchQuery,
backend: &str,
@@ -52,8 +65,7 @@ pub async fn search_channel_databuffer(
&backend,
],
)
.await
.err_conv()?;
.await?;
let mut res = Vec::new();
for row in rows {
let shapedb: Option<serde_json::Value> = row.get(4);
@@ -68,14 +80,14 @@ pub async fn search_channel_databuffer(
Some(n) => {
a.push(n as u32);
}
None => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
None => return Err(Error::BadShapeFromDb(format!("{:?}", shapedb))),
},
_ => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
_ => return Err(Error::BadShapeFromDb(format!("{:?}", shapedb))),
}
}
a
}
_ => return Err(Error::with_msg(format!("can not understand shape {:?}", shapedb))),
_ => return Err(Error::BadShapeFromDb(format!("{:?}", shapedb))),
},
None => Vec::new(),
};
@@ -131,7 +143,7 @@ pub(super) async fn search_channel_scylla(
regop
);
let params: &[&(dyn tokio_postgres::types::ToSql + Sync)] = &[&ch_kind, &query.name_regex, &cb1, &cb2];
info!("search_channel_scylla {:?}", params);
trace!("search_channel_scylla {:?}", params);
let rows = pgc.query(sql, params).await.err_conv()?;
let mut res = Vec::new();
for row in rows {
@@ -227,10 +239,7 @@ async fn search_channel_archeng(
if k == "Scalar" {
Vec::new()
} else {
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
return Err(Error::BadArchEngValue(format!("{:?}", config)));
}
}
JsVal::Object(k) => match k.get("Wave") {
@@ -239,24 +248,15 @@ async fn search_channel_archeng(
vec![k.as_i64().unwrap_or(u32::MAX as i64) as u32]
}
_ => {
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
return Err(Error::BadArchEngValue(format!("{:?}", config)));
}
},
None => {
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
return Err(Error::BadArchEngValue(format!("{:?}", config)));
}
},
_ => {
return Err(Error::with_msg_no_trace(format!(
"search_channel_archeng can not understand {:?}",
config
)));
return Err(Error::BadArchEngValue(format!("{:?}", config)));
}
},
None => Vec::new(),
@@ -290,16 +290,12 @@ pub async fn search_channel(
let mut query = query;
query.backend = Some(backend.into());
if let Some(_scyconf) = ncc.node_config.cluster.scylla_st() {
pgqueue
.search_channel_scylla(query)
.await
.map_err(|e| Error::with_msg_no_trace(format!("db worker error {e}")))?
pgqueue.search_channel_scylla(query).await?
// search_channel_scylla(query, backend, pgconf).await
} else if let Some(conf) = ncc.node.channel_archiver.as_ref() {
search_channel_archeng(query, backend.clone(), conf, pgconf).await
} else if let Some(_conf) = ncc.node.archiver_appliance.as_ref() {
// TODO
err::todoval()
todo!()
} else {
search_channel_databuffer(query, backend, ncc).await
}
crates/dbconn/src/worker.rs
+17 -15
View File
@@ -3,29 +3,28 @@ use async_channel::Receiver;
use async_channel::RecvError;
use async_channel::Sender;
use daqbuf_err as err;
use err::thiserror;
use err::ThisError;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use netpod::ChConf;
use netpod::ChannelSearchQuery;
use netpod::ChannelSearchResult;
use netpod::Database;
use netpod::SeriesKind;
use netpod::SfDbChannel;
use netpod::log::*;
use netpod::range::evrange::NanoRange;
use taskrun::tokio;
use tokio::task::JoinHandle;
use tokio_postgres::Client;
#[derive(Debug, ThisError)]
#[cstm(name = "PgWorker")]
pub enum Error {
Error(#[from] err::Error),
ChannelSend,
ChannelRecv,
Join,
ChannelConfig(#[from] crate::channelconfig::Error),
}
autoerr::create_error_v1!(
name(Error, "PgWorker"),
enum variants {
Error(#[from] err::Error),
ChannelSend,
ChannelRecv,
Join,
ChannelConfig(#[from] crate::channelconfig::Error),
},
);
impl From<RecvError> for Error {
fn from(_value: RecvError) -> Self {
@@ -51,7 +50,10 @@ enum Job {
Vec<u64>,
Sender<Result<Vec<Option<crate::channelinfo::ChannelInfo>>, crate::channelinfo::Error>>,
),
SearchChannel(ChannelSearchQuery, Sender<Result<ChannelSearchResult, err::Error>>),
SearchChannel(
ChannelSearchQuery,
Sender<Result<ChannelSearchResult, crate::search::Error>>,
),
SfChannelBySeries(
netpod::SfDbChannel,
Sender<Result<netpod::SfDbChannel, crate::FindChannelError>>,
@@ -101,7 +103,7 @@ impl PgQueue {
pub async fn search_channel_scylla(
&self,
query: ChannelSearchQuery,
) -> Result<Result<ChannelSearchResult, err::Error>, Error> {
) -> Result<Result<ChannelSearchResult, crate::search::Error>, Error> {
let (tx, rx) = async_channel::bounded(1);
let job = Job::SearchChannel(query, tx);
self.tx.send(job).await.map_err(|_| Error::ChannelSend)?;