daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-11-04 12:07:15 +01:00
parent 66538ced79
commit 5543389c1e
9 changed files with 202 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
crates/query/src/api4.rs
View File

@@ -4,7 +4,6 @@ pub mod events;
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::range::evrange::SeriesRange;
use netpod::ttl::RetentionTime;
@@ -21,6 +20,16 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Query")]
pub enum Error {
MissingTimerange,
ChronoParse(#[from] chrono::ParseError),
HumantimeDurationParse(#[from] humantime::DurationError),
MissingBackend,
MissingRetentionTime,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AccountingIngestedBytesQuery {
backend: String,
@@ -46,16 +55,18 @@ impl HasTimeout for AccountingIngestedBytesQuery {
}
impl FromUrl for AccountingIngestedBytesQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
type Error = netpod::NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = Self {
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| netpod::NetpodError::MissingBackend)?
.to_string(),
range: SeriesRange::from_pairs(pairs)?,
};
@@ -115,36 +126,32 @@ impl HasTimeout for AccountingToplistQuery {
}
impl FromUrl for AccountingToplistQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let fn1 = |pairs: &BTreeMap<String, String>| {
let v = pairs
.get("tsDate")
.ok_or(Error::with_public_msg_no_trace("missing tsDate"))?;
let v = pairs.get("tsDate").ok_or(Self::Error::MissingTimerange)?;
let mut w = v.parse::<DateTime<Utc>>();
if w.is_err() && v.ends_with("ago") {
let d = humantime::parse_duration(&v[..v.len() - 3])
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse {v}")))?;
let d = humantime::parse_duration(&v[..v.len() - 3])?;
w = Ok(Utc::now() - d);
}
let w = w?;
Ok::<_, Error>(TsNano::from_ns(w.to_nanos()))
Ok::<_, Self::Error>(TsNano::from_ns(w.to_nanos()))
};
let ret = Self {
rt: pairs
.get("retentionTime")
.ok_or_else(|| Error::with_public_msg_no_trace("missing retentionTime"))
.and_then(|x| {
x.parse()
.map_err(|_| Error::with_public_msg_no_trace("missing retentionTime"))
})?,
.ok_or_else(|| Self::Error::MissingRetentionTime)
.and_then(|x| x.parse().map_err(|_| Self::Error::MissingRetentionTime))?,
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| Self::Error::MissingBackend)?
.to_string(),
ts: fn1(pairs)?,
limit: pairs.get("limit").map_or(None, |x| x.parse().ok()).unwrap_or(20),

27
crates/query/src/api4/binned.rs
View File

@@ -1,5 +1,4 @@
use crate::transform::TransformQuery;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::query::CacheUsage;
@@ -20,6 +19,14 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "BinnedQuery")]
pub enum Error {
MultipleBinCountBinWidth,
BadUseRt,
Netpod(#[from] netpod::NetpodError),
}
mod serde_option_vec_duration {
use serde::Deserialize;
use serde::Deserializer;
@@ -222,9 +229,7 @@ impl BinnedQuery {
SeriesRange::TimeRange(range) => match self.bin_width {
Some(dt) => {
if self.bin_count.is_some() {
Err(Error::with_public_msg_no_trace(format!(
"must not specify both binWidth and binCount"
)))
Err(Error::MultipleBinCountBinWidth)
} else {
let ret = BinnedRangeEnum::Time(BinnedRange::covering_range_time(
range.clone(),
@@ -257,12 +262,14 @@ impl HasTimeout for BinnedQuery {
}
impl FromUrl for BinnedQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = Self {
channel: SfDbChannel::from_pairs(&pairs)?,
range: SeriesRange::from_pairs(pairs)?,
@@ -297,11 +304,9 @@ impl FromUrl for BinnedQuery {
.map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
test_do_wasm: pairs.get("testDoWasm").map(|x| String::from(x)),
log_level: pairs.get("log_level").map_or(String::new(), String::from),
use_rt: pairs.get("useRt").map_or(Ok(None), |k| {
k.parse()
.map(Some)
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse useRt: {}", k)))
})?,
use_rt: pairs
.get("useRt")
.map_or(Ok(None), |k| k.parse().map(Some).map_err(|_| Error::BadUseRt))?,
};
debug!("BinnedQuery::from_url {:?}", ret);
Ok(ret)

31
crates/query/src/api4/events.rs
View File

@@ -1,6 +1,5 @@
use super::binned::BinnedQuery;
use crate::transform::TransformQuery;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::is_false;
use netpod::query::api1::Api1Query;
@@ -22,6 +21,16 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "EventsQuery")]
pub enum Error {
BadInt(#[from] std::num::ParseIntError),
MissingTimerange,
BadQuery,
Transform(#[from] crate::transform::Error),
Netpod(#[from] netpod::NetpodError),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PlainEventsQuery {
channel: SfDbChannel,
@@ -246,18 +255,20 @@ impl HasTimeout for PlainEventsQuery {
}
impl FromUrl for PlainEventsQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let range = if let Ok(x) = TimeRangeQuery::from_pairs(pairs) {
SeriesRange::TimeRange(x.into())
} else if let Ok(x) = PulseRangeQuery::from_pairs(pairs) {
SeriesRange::PulseRange(x.into())
} else {
return Err(Error::with_public_msg_no_trace("no time range in url"));
return Err(Error::MissingTimerange);
};
let ret = Self {
channel: SfDbChannel::from_pairs(pairs)?,
@@ -284,12 +295,12 @@ impl FromUrl for PlainEventsQuery {
.get("doTestMainError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_public_msg_no_trace(format!("can not parse doTestMainError: {}", e)))?,
.map_err(|_| Error::BadQuery)?,
do_test_stream_error: pairs
.get("doTestStreamError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_public_msg_no_trace(format!("can not parse doTestStreamError: {}", e)))?,
.map_err(|_| Error::BadQuery)?,
// test_do_wasm: pairs
// .get("testDoWasm")
// .map(|x| x.parse::<bool>().ok())
@@ -307,11 +318,9 @@ impl FromUrl for PlainEventsQuery {
.map(|x| x.split(",").map(|x| x.to_string()).collect())
.unwrap_or(Vec::new()),
log_level: pairs.get("log_level").map_or(String::new(), String::from),
use_rt: pairs.get("useRt").map_or(Ok(None), |k| {
k.parse()
.map(Some)
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse useRt: {}", k)))
})?,
use_rt: pairs
.get("useRt")
.map_or(Ok(None), |k| k.parse().map(Some).map_err(|_| Error::BadQuery))?,
querymarker: pairs.get("querymarker").map_or(String::new(), |x| x.to_string()),
};
Ok(ret)

23
crates/query/src/transform.rs
View File

@@ -1,4 +1,3 @@
use err::Error;
use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::AppendToUrl;
@@ -6,8 +5,17 @@ use netpod::FromUrl;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
use thiserror;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Query")]
pub enum Error {
ParseInt(#[from] std::num::ParseIntError),
BadEnumAsString,
BadBinningScheme,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum EventTransformQuery {
EventBlobsVerbatim,
@@ -155,17 +163,16 @@ impl TransformQuery {
}
impl FromUrl for TransformQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let enum_as_string = if let Some(k) = pairs.get("enumAsString") {
Some(
k.parse()
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse enumAsString: {}", k)))?,
)
Some(k.parse().map_err(|_| Error::BadEnumAsString)?)
} else {
None
};
@@ -217,7 +224,7 @@ impl FromUrl for TransformQuery {
enum_as_string,
}
} else {
return Err(Error::with_msg("can not extract binningScheme"));
return Err(Error::BadBinningScheme);
};
Ok(ret)
} else {