daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2024-11-04 12:07:15 +01:00
parent 66538ced79
commit 5543389c1e
9 changed files with 202 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
crates/netpod/src/netpod.rs
View File

@@ -181,7 +181,33 @@ impl CmpZero for usize {
}
}
#[derive(Debug, err::ThisError)]
#[derive(Debug, ThisError)]
#[cstm(name = "Netpod")]
pub enum NetpodError {
UnknownSeriesKind(i64),
BadInt(#[from] std::num::ParseIntError),
ChronoParse(#[from] chrono::ParseError),
HumantimeDurationParse(#[from] humantime::DurationError),
MissingQueryParameters,
MissingSeries,
MissingBackend,
MissingTimerange,
BadTimerange,
NoSeriesNoName,
BadScalarTypeIndex(i64),
UnsupportedDtype(u8),
JsonParse(#[from] serde_json::Error),
BadScalarTypeVariant(String),
BadScalarTypeCaId(u16),
ScalarTypeNotInCa,
MissingScalarType,
MissingShape,
MissingBinningScheme,
BadCacheUsage(String),
TimelikeBinWidthImpossibleForPulseRange,
}
#[derive(Debug, ThisError)]
#[cstm(name = "AsyncChannelError")]
pub enum AsyncChannelError {
Send,
@@ -212,12 +238,12 @@ impl SeriesKind {
}
}
pub fn from_db_i16(x: i16) -> Result<Self, Error> {
pub fn from_db_i16(x: i16) -> Result<Self, NetpodError> {
let ret = match x {
1 => Self::ChannelStatus,
2 => Self::ChannelData,
3 => Self::CaStatus,
_ => return Err(Error::with_msg_no_trace("bad SeriesKind value")),
_ => return Err(NetpodError::UnknownSeriesKind(x as i64)),
};
Ok(ret)
}
@@ -230,12 +256,14 @@ impl Default for SeriesKind {
}
impl FromUrl for SeriesKind {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = pairs
.get("seriesKind")
.and_then(|x| match x.as_str() {
@@ -360,7 +388,7 @@ pub trait HasScalarType {
}
impl ScalarType {
pub fn from_dtype_index(ix: u8) -> Result<Self, Error> {
pub fn from_dtype_index(ix: u8) -> Result<Self, NetpodError> {
use ScalarType::*;
let g = match ix {
0 => BOOL,
@@ -377,8 +405,7 @@ impl ScalarType {
12 => F64,
13 => STRING,
15 => Enum,
6 => return Err(Error::with_msg(format!("CHARACTER not supported"))),
_ => return Err(Error::with_msg(format!("unknown dtype code: {:?}", ix))),
_ => return Err(NetpodError::UnsupportedDtype(ix)),
};
Ok(g)
}
@@ -402,7 +429,7 @@ impl ScalarType {
}
}
pub fn from_variant_str(s: &str) -> Result<Self, Error> {
pub fn from_variant_str(s: &str) -> Result<Self, NetpodError> {
use ScalarType::*;
let ret = match s {
"u8" => U8,
@@ -418,12 +445,7 @@ impl ScalarType {
"bool" => BOOL,
"string" => STRING,
"enum" => Enum,
_ => {
return Err(Error::with_msg_no_trace(format!(
"from_bsread_str can not understand bsread {:?}",
s
)))
}
_ => return Err(NetpodError::BadScalarTypeVariant(s.into())),
};
Ok(ret)
}
@@ -447,7 +469,7 @@ impl ScalarType {
}
}
pub fn from_bsread_str(s: &str) -> Result<Self, Error> {
pub fn from_bsread_str(s: &str) -> Result<Self, NetpodError> {
use ScalarType::*;
let ret = match s {
"uint8" => U8,
@@ -465,17 +487,12 @@ impl ScalarType {
"bool" => BOOL,
"string" => STRING,
"enum" => Enum,
_ => {
return Err(Error::with_msg_no_trace(format!(
"from_bsread_str can not understand bsread {:?}",
s
)))
}
_ => return Err(NetpodError::BadScalarTypeVariant(s.into())),
};
Ok(ret)
}
pub fn from_ca_id(k: u16) -> Result<Self, Error> {
pub fn from_ca_id(k: u16) -> Result<Self, NetpodError> {
use ScalarType::*;
let ret = match k {
0 => STRING,
@@ -485,17 +502,12 @@ impl ScalarType {
4 => I8,
5 => I32,
6 => F64,
_ => {
return Err(Error::with_msg_no_trace(format!(
"from_ca_id can not understand {:?}",
k
)))
}
_ => return Err(NetpodError::BadScalarTypeCaId(k)),
};
Ok(ret)
}
pub fn to_ca_id(&self) -> Result<u16, Error> {
pub fn to_ca_id(&self) -> Result<u16, NetpodError> {
use ScalarType::*;
let ret = match self {
I8 => 4,
@@ -505,12 +517,12 @@ impl ScalarType {
F64 => 6,
STRING => 0,
Enum => 3,
_ => return Err(Error::with_msg_no_trace(format!("can not represent {self:?} as CA id"))),
_ => return Err(NetpodError::ScalarTypeNotInCa),
};
Ok(ret)
}
pub fn from_archeng_db_str(s: &str) -> Result<Self, Error> {
pub fn from_archeng_db_str(s: &str) -> Result<Self, NetpodError> {
use ScalarType::*;
let ret = match s {
"I8" => I8,
@@ -519,19 +531,14 @@ impl ScalarType {
"I64" => I64,
"F32" => F32,
"F64" => F64,
_ => {
return Err(Error::with_msg_no_trace(format!(
"from_archeng_db_str can not understand {:?}",
s
)))
}
_ => return Err(NetpodError::BadScalarTypeVariant(s.into())),
};
Ok(ret)
}
pub fn from_scylla_i32(k: i32) -> Result<Self, Error> {
pub fn from_scylla_i32(k: i32) -> Result<Self, NetpodError> {
if k < 0 || k > u8::MAX as i32 {
return Err(Error::with_public_msg_no_trace(format!("bad scalar type index {k}")));
return Err(NetpodError::BadScalarTypeIndex(k as i64));
}
Self::from_dtype_index(k as u8)
}
@@ -583,7 +590,7 @@ impl ScalarType {
self.index() as i32
}
pub fn from_url_str(s: &str) -> Result<Self, Error> {
pub fn from_url_str(s: &str) -> Result<Self, NetpodError> {
let ret = serde_json::from_str(&format!("\"{s}\""))?;
Ok(ret)
}
@@ -1120,17 +1127,16 @@ impl fmt::Display for SfDbChannel {
}
impl FromUrl for SfDbChannel {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = SfDbChannel {
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.into(),
backend: pairs.get("backend").ok_or_else(|| NetpodError::MissingBackend)?.into(),
name: pairs
.get("channelName")
.map(String::from)
@@ -1142,9 +1148,7 @@ impl FromUrl for SfDbChannel {
kind: SeriesKind::from_pairs(pairs)?,
};
if ret.name.is_empty() && ret.series.is_none() {
return Err(Error::with_public_msg_no_trace(format!(
"Missing one of channelName or seriesId parameters."
)));
return Err(NetpodError::NoSeriesNoName);
}
Ok(ret)
}
@@ -1207,21 +1211,20 @@ impl DaqbufSeries {
}
impl FromUrl for DaqbufSeries {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = DaqbufSeries {
series: pairs
.get("seriesId")
.ok_or_else(|| Error::with_public_msg_no_trace("missing seriesId"))
.ok_or_else(|| NetpodError::MissingSeries)
.map(|x| x.parse::<u64>())??,
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.into(),
backend: pairs.get("backend").ok_or_else(|| NetpodError::MissingBackend)?.into(),
name: pairs
.get("channelName")
.map(String::from)
@@ -1593,7 +1596,7 @@ impl Shape {
}
}
pub fn from_url_str(s: &str) -> Result<Self, Error> {
pub fn from_url_str(s: &str) -> Result<Self, NetpodError> {
let ret = serde_json::from_str(s)?;
Ok(ret)
}
@@ -2304,11 +2307,13 @@ impl PreBinnedPatchCoordEnum {
}
impl FromUrl for PreBinnedPatchCoordEnum {
fn from_url(_url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(_url: &Url) -> Result<Self, Self::Error> {
todo!()
}
fn from_pairs(_pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(_pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
todo!()
}
}
@@ -2442,7 +2447,7 @@ impl BinnedRange<TsNano> {
}
}
pub fn covering_range_time(range: NanoRange, bin_len_req: DtMs) -> Result<Self, Error> {
pub fn covering_range_time(range: NanoRange, bin_len_req: DtMs) -> Result<Self, NetpodError> {
let opts = <TsNano as Dim0Index>::binned_bin_len_opts();
let bin_len_req = if bin_len_req.ms() < opts[0].ms() {
DtMs::from_ms_u64(opts[0].ms())
@@ -2598,12 +2603,10 @@ impl BinnedRangeEnum {
}
/// Cover at least the given range while selecting the bin width which best fits the requested bin width.
pub fn covering_range_time(range: SeriesRange, bin_len_req: DtMs) -> Result<Self, Error> {
pub fn covering_range_time(range: SeriesRange, bin_len_req: DtMs) -> Result<Self, NetpodError> {
match range {
SeriesRange::TimeRange(k) => Ok(Self::Time(BinnedRange::covering_range_time(k, bin_len_req)?)),
SeriesRange::PulseRange(_) => Err(Error::with_msg_no_trace(format!(
"timelike bin width not possible for a pulse range"
))),
SeriesRange::PulseRange(_) => Err(NetpodError::TimelikeBinWidthImpossibleForPulseRange),
}
}
@@ -3222,11 +3225,13 @@ impl Default for DiskIoTune {
}
impl FromUrl for DiskIoTune {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
Self::from_pairs(&get_url_query_pairs(url))
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let read_sys = pairs
.get("ReadSys")
.map(|x| x.as_str().into())
@@ -3263,7 +3268,7 @@ pub struct ChannelSearchQuery {
}
impl ChannelSearchQuery {
pub fn from_url(url: &Url) -> Result<Self, Error> {
pub fn from_url(url: &Url) -> Result<Self, NetpodError> {
let pairs = get_url_query_pairs(url);
let ret = Self {
backend: pairs.get("backend").map(Into::into),
@@ -3358,9 +3363,10 @@ pub trait HasTimeout {
}
pub trait FromUrl: Sized {
fn from_url(url: &Url) -> Result<Self, Error>;
type Error;
fn from_url(url: &Url) -> Result<Self, Self::Error>;
// TODO put this in separate trait, because some implementors need url path segments to construct.
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error>;
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error>;
}
pub trait AppendToUrl {
@@ -3379,12 +3385,14 @@ impl AppendToUrl for MapQuery {
}
impl FromUrl for MapQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
Ok(pairs.clone())
}
}
@@ -3434,12 +3442,14 @@ impl HasTimeout for ChannelConfigQuery {
}
impl FromUrl for ChannelConfigQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let beg_date = pairs
.get("begDate")
.map(String::from)

48
crates/netpod/src/query.rs
View File

@@ -10,6 +10,7 @@ use crate::FromUrl;
use crate::HasBackend;
use crate::HasTimeout;
use crate::NanoRange;
use crate::NetpodError;
use crate::PulseRange;
use crate::SfDbChannel;
use crate::ToNanos;
@@ -45,7 +46,7 @@ impl CacheUsage {
}
// Missing query parameter is not an error
pub fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Option<Self>, Error> {
pub fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Option<Self>, NetpodError> {
pairs
.get("cacheUsage")
.map(|k| {
@@ -58,7 +59,7 @@ impl CacheUsage {
} else if k == "v0nocache" {
Ok(Some(CacheUsage::V0NoCache))
} else {
Err(Error::with_msg(format!("unexpected cacheUsage {:?}", k)))?
Err(NetpodError::BadCacheUsage(k.clone()))?
}
})
.unwrap_or(Ok(None))
@@ -109,27 +110,28 @@ pub struct TimeRangeQuery {
range: NanoRange,
}
fn parse_time(v: &str) -> Result<DateTime<Utc>, Error> {
fn parse_time(v: &str) -> Result<DateTime<Utc>, NetpodError> {
if let Ok(x) = v.parse() {
Ok(x)
} else {
if v.ends_with("ago") {
let d = humantime::parse_duration(&v[..v.len() - 3])
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse {v}")))?;
let d = humantime::parse_duration(&v[..v.len() - 3]).map_err(|_| NetpodError::BadTimerange)?;
Ok(Utc::now() - d)
} else {
Err(Error::with_public_msg_no_trace(format!("can not parse {v}")))
Err(NetpodError::BadTimerange)
}
}
}
impl FromUrl for TimeRangeQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
if let (Some(beg), Some(end)) = (pairs.get("begDate"), pairs.get("endDate")) {
let ret = Self {
range: NanoRange {
@@ -147,7 +149,7 @@ impl FromUrl for TimeRangeQuery {
};
Ok(ret)
} else {
Err(Error::with_public_msg_no_trace("missing date range"))
Err(NetpodError::MissingTimerange)
}
}
}
@@ -198,12 +200,14 @@ pub struct PulseRangeQuery {
}
impl FromUrl for PulseRangeQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
if let (Some(beg), Some(end)) = (pairs.get("begPulse"), pairs.get("endPulse")) {
let ret = Self {
range: PulseRange {
@@ -213,7 +217,7 @@ impl FromUrl for PulseRangeQuery {
};
Ok(ret)
} else {
Err(Error::with_public_msg_no_trace("missing pulse range"))
Err(NetpodError::MissingQueryParameters)
}
}
}
@@ -261,7 +265,7 @@ pub fn binning_scheme_append_to_url(agg_kind: &AggKind, url: &mut Url) {
}
// Absent AggKind is not considered an error.
pub fn agg_kind_from_binning_scheme(pairs: &BTreeMap<String, String>) -> Result<Option<AggKind>, Error> {
pub fn agg_kind_from_binning_scheme(pairs: &BTreeMap<String, String>) -> Result<Option<AggKind>, NetpodError> {
let key = "binningScheme";
if let Some(s) = pairs.get(key) {
let ret = if s == "eventBlobs" {
@@ -278,7 +282,7 @@ pub fn agg_kind_from_binning_scheme(pairs: &BTreeMap<String, String>) -> Result<
} else if s == "pulseIdDiff" {
AggKind::PulseIdDiff
} else {
return Err(Error::with_msg("can not extract binningScheme"));
return Err(NetpodError::MissingBinningScheme);
};
Ok(Some(ret))
} else {
@@ -327,18 +331,16 @@ impl HasTimeout for ChannelStateEventsQuery {
}
impl FromUrl for ChannelStateEventsQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
let beg_date = pairs
.get("begDate")
.ok_or_else(|| Error::with_msg_no_trace("missing begDate"))?;
let end_date = pairs
.get("endDate")
.ok_or_else(|| Error::with_msg_no_trace("missing endDate"))?;
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let beg_date = pairs.get("begDate").ok_or_else(|| NetpodError::MissingTimerange)?;
let end_date = pairs.get("endDate").ok_or_else(|| NetpodError::MissingTimerange)?;
let ret = Self {
channel: SfDbChannel::from_pairs(&pairs)?,
range: NanoRange {
@@ -347,7 +349,7 @@ impl FromUrl for ChannelStateEventsQuery {
},
};
let self_name = std::any::type_name::<Self>();
info!("{self_name}::from_url {ret:?}");
debug!("{self_name}::from_url {ret:?}");
Ok(ret)
}
}

11
crates/netpod/src/query/prebinned.rs
View File

@@ -5,11 +5,11 @@ use crate::AggKind;
use crate::AppendToUrl;
use crate::ByteSize;
use crate::FromUrl;
use crate::NetpodError;
use crate::PreBinnedPatchCoordEnum;
use crate::ScalarType;
use crate::SfDbChannel;
use crate::Shape;
use err::Error;
use std::collections::BTreeMap;
use url::Url;
@@ -48,7 +48,7 @@ impl PreBinnedQuery {
}
}
pub fn from_url(url: &Url) -> Result<Self, Error> {
pub fn from_url(url: &Url) -> Result<Self, NetpodError> {
let mut pairs = BTreeMap::new();
for (j, k) in url.query_pairs() {
pairs.insert(j.to_string(), k.to_string());
@@ -56,11 +56,11 @@ impl PreBinnedQuery {
let pairs = pairs;
let scalar_type = pairs
.get("scalarType")
.ok_or_else(|| Error::with_msg("missing scalarType"))
.ok_or_else(|| NetpodError::MissingScalarType)
.map(|x| ScalarType::from_url_str(&x))??;
let shape = pairs
.get("shape")
.ok_or_else(|| Error::with_msg("missing shape"))
.ok_or_else(|| NetpodError::MissingShape)
.map(|x| Shape::from_url_str(&x))??;
let ret = Self {
patch: PreBinnedPatchCoordEnum::from_pairs(&pairs)?,
@@ -119,6 +119,9 @@ impl PreBinnedQuery {
impl AppendToUrl for PreBinnedQuery {
fn append_to_url(&self, url: &mut Url) {
if false {
panic!("remove, not in use");
}
self.patch.append_to_url(url);
self.channel.append_to_url(url);
self.shape.append_to_url(url);

9
crates/netpod/src/range/evrange.rs
View File

@@ -4,6 +4,7 @@ use crate::timeunits::SEC;
use crate::AppendToUrl;
use crate::Dim0Kind;
use crate::FromUrl;
use crate::NetpodError;
use crate::TsNano;
use chrono::DateTime;
use chrono::TimeZone;
@@ -189,18 +190,20 @@ impl From<PulseRange> for SeriesRange {
}
impl FromUrl for SeriesRange {
fn from_url(url: &url::Url) -> Result<Self, Error> {
type Error = NetpodError;
fn from_url(url: &url::Url) -> Result<Self, Self::Error> {
let pairs = crate::get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = if let Ok(x) = TimeRangeQuery::from_pairs(pairs) {
SeriesRange::TimeRange(x.into())
} else if let Ok(x) = PulseRangeQuery::from_pairs(pairs) {
SeriesRange::PulseRange(x.into())
} else {
return Err(Error::with_public_msg_no_trace("no time range in url"));
return Err(NetpodError::MissingTimerange);
};
Ok(ret)
}

5
crates/query/Cargo.toml
View File

@@ -12,7 +12,10 @@ chrono = { version = "0.4.19", features = ["serde"] }
url = "2.2"
humantime = "2.1.0"
humantime-serde = "1.1.1"
err = { path = "../err" }
thiserror = "1"
netpod = { path = "../netpod" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }
[patch.crates-io]
thiserror = { git = "https://github.com/dominikwerder/thiserror.git", branch = "cstm" }

43
crates/query/src/api4.rs
View File

@@ -4,7 +4,6 @@ pub mod events;
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::range::evrange::SeriesRange;
use netpod::ttl::RetentionTime;
@@ -21,6 +20,16 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Query")]
pub enum Error {
MissingTimerange,
ChronoParse(#[from] chrono::ParseError),
HumantimeDurationParse(#[from] humantime::DurationError),
MissingBackend,
MissingRetentionTime,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AccountingIngestedBytesQuery {
backend: String,
@@ -46,16 +55,18 @@ impl HasTimeout for AccountingIngestedBytesQuery {
}
impl FromUrl for AccountingIngestedBytesQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
type Error = netpod::NetpodError;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = Self {
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| netpod::NetpodError::MissingBackend)?
.to_string(),
range: SeriesRange::from_pairs(pairs)?,
};
@@ -115,36 +126,32 @@ impl HasTimeout for AccountingToplistQuery {
}
impl FromUrl for AccountingToplistQuery {
fn from_url(url: &Url) -> Result<Self, err::Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let fn1 = |pairs: &BTreeMap<String, String>| {
let v = pairs
.get("tsDate")
.ok_or(Error::with_public_msg_no_trace("missing tsDate"))?;
let v = pairs.get("tsDate").ok_or(Self::Error::MissingTimerange)?;
let mut w = v.parse::<DateTime<Utc>>();
if w.is_err() && v.ends_with("ago") {
let d = humantime::parse_duration(&v[..v.len() - 3])
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse {v}")))?;
let d = humantime::parse_duration(&v[..v.len() - 3])?;
w = Ok(Utc::now() - d);
}
let w = w?;
Ok::<_, Error>(TsNano::from_ns(w.to_nanos()))
Ok::<_, Self::Error>(TsNano::from_ns(w.to_nanos()))
};
let ret = Self {
rt: pairs
.get("retentionTime")
.ok_or_else(|| Error::with_public_msg_no_trace("missing retentionTime"))
.and_then(|x| {
x.parse()
.map_err(|_| Error::with_public_msg_no_trace("missing retentionTime"))
})?,
.ok_or_else(|| Self::Error::MissingRetentionTime)
.and_then(|x| x.parse().map_err(|_| Self::Error::MissingRetentionTime))?,
backend: pairs
.get("backend")
.ok_or_else(|| Error::with_public_msg_no_trace("missing backend"))?
.ok_or_else(|| Self::Error::MissingBackend)?
.to_string(),
ts: fn1(pairs)?,
limit: pairs.get("limit").map_or(None, |x| x.parse().ok()).unwrap_or(20),

27
crates/query/src/api4/binned.rs
View File

@@ -1,5 +1,4 @@
use crate::transform::TransformQuery;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::query::CacheUsage;
@@ -20,6 +19,14 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "BinnedQuery")]
pub enum Error {
MultipleBinCountBinWidth,
BadUseRt,
Netpod(#[from] netpod::NetpodError),
}
mod serde_option_vec_duration {
use serde::Deserialize;
use serde::Deserializer;
@@ -222,9 +229,7 @@ impl BinnedQuery {
SeriesRange::TimeRange(range) => match self.bin_width {
Some(dt) => {
if self.bin_count.is_some() {
Err(Error::with_public_msg_no_trace(format!(
"must not specify both binWidth and binCount"
)))
Err(Error::MultipleBinCountBinWidth)
} else {
let ret = BinnedRangeEnum::Time(BinnedRange::covering_range_time(
range.clone(),
@@ -257,12 +262,14 @@ impl HasTimeout for BinnedQuery {
}
impl FromUrl for BinnedQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let ret = Self {
channel: SfDbChannel::from_pairs(&pairs)?,
range: SeriesRange::from_pairs(pairs)?,
@@ -297,11 +304,9 @@ impl FromUrl for BinnedQuery {
.map_or(Ok(None), |k| k.parse().map(|k| Some(k)))?,
test_do_wasm: pairs.get("testDoWasm").map(|x| String::from(x)),
log_level: pairs.get("log_level").map_or(String::new(), String::from),
use_rt: pairs.get("useRt").map_or(Ok(None), |k| {
k.parse()
.map(Some)
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse useRt: {}", k)))
})?,
use_rt: pairs
.get("useRt")
.map_or(Ok(None), |k| k.parse().map(Some).map_err(|_| Error::BadUseRt))?,
};
debug!("BinnedQuery::from_url {:?}", ret);
Ok(ret)

31
crates/query/src/api4/events.rs
View File

@@ -1,6 +1,5 @@
use super::binned::BinnedQuery;
use crate::transform::TransformQuery;
use err::Error;
use netpod::get_url_query_pairs;
use netpod::is_false;
use netpod::query::api1::Api1Query;
@@ -22,6 +21,16 @@ use std::collections::BTreeMap;
use std::time::Duration;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "EventsQuery")]
pub enum Error {
BadInt(#[from] std::num::ParseIntError),
MissingTimerange,
BadQuery,
Transform(#[from] crate::transform::Error),
Netpod(#[from] netpod::NetpodError),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PlainEventsQuery {
channel: SfDbChannel,
@@ -246,18 +255,20 @@ impl HasTimeout for PlainEventsQuery {
}
impl FromUrl for PlainEventsQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let range = if let Ok(x) = TimeRangeQuery::from_pairs(pairs) {
SeriesRange::TimeRange(x.into())
} else if let Ok(x) = PulseRangeQuery::from_pairs(pairs) {
SeriesRange::PulseRange(x.into())
} else {
return Err(Error::with_public_msg_no_trace("no time range in url"));
return Err(Error::MissingTimerange);
};
let ret = Self {
channel: SfDbChannel::from_pairs(pairs)?,
@@ -284,12 +295,12 @@ impl FromUrl for PlainEventsQuery {
.get("doTestMainError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_public_msg_no_trace(format!("can not parse doTestMainError: {}", e)))?,
.map_err(|_| Error::BadQuery)?,
do_test_stream_error: pairs
.get("doTestStreamError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_public_msg_no_trace(format!("can not parse doTestStreamError: {}", e)))?,
.map_err(|_| Error::BadQuery)?,
// test_do_wasm: pairs
// .get("testDoWasm")
// .map(|x| x.parse::<bool>().ok())
@@ -307,11 +318,9 @@ impl FromUrl for PlainEventsQuery {
.map(|x| x.split(",").map(|x| x.to_string()).collect())
.unwrap_or(Vec::new()),
log_level: pairs.get("log_level").map_or(String::new(), String::from),
use_rt: pairs.get("useRt").map_or(Ok(None), |k| {
k.parse()
.map(Some)
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse useRt: {}", k)))
})?,
use_rt: pairs
.get("useRt")
.map_or(Ok(None), |k| k.parse().map(Some).map_err(|_| Error::BadQuery))?,
querymarker: pairs.get("querymarker").map_or(String::new(), |x| x.to_string()),
};
Ok(ret)

23
crates/query/src/transform.rs
View File

@@ -1,4 +1,3 @@
use err::Error;
use netpod::get_url_query_pairs;
use netpod::log::*;
use netpod::AppendToUrl;
@@ -6,8 +5,17 @@ use netpod::FromUrl;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
use thiserror;
use url::Url;
#[derive(Debug, thiserror::Error)]
#[cstm(name = "Query")]
pub enum Error {
ParseInt(#[from] std::num::ParseIntError),
BadEnumAsString,
BadBinningScheme,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum EventTransformQuery {
EventBlobsVerbatim,
@@ -155,17 +163,16 @@ impl TransformQuery {
}
impl FromUrl for TransformQuery {
fn from_url(url: &Url) -> Result<Self, Error> {
type Error = Error;
fn from_url(url: &Url) -> Result<Self, Self::Error> {
let pairs = get_url_query_pairs(url);
Self::from_pairs(&pairs)
}
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Error> {
fn from_pairs(pairs: &BTreeMap<String, String>) -> Result<Self, Self::Error> {
let enum_as_string = if let Some(k) = pairs.get("enumAsString") {
Some(
k.parse()
.map_err(|_| Error::with_public_msg_no_trace(format!("can not parse enumAsString: {}", k)))?,
)
Some(k.parse().map_err(|_| Error::BadEnumAsString)?)
} else {
None
};
@@ -217,7 +224,7 @@ impl FromUrl for TransformQuery {
enum_as_string,
}
} else {
return Err(Error::with_msg("can not extract binningScheme"));
return Err(Error::BadBinningScheme);
};
Ok(ret)
} else {