daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First basic plain event fetch as binary

Browse Source
This commit is contained in:
Dominik Werder
2021-06-10 17:37:40 +02:00
parent 99c45985ef
commit 0c43d5367e
9 changed files with 499 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
disk/src/binned.rs
View File

@@ -12,7 +12,7 @@ use crate::decode::{
LittleEndian, NumFromBytes,
};
use crate::frame::makeframe::{Framable, FrameType, SubFrId};
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use crate::Sitemty;
use bytes::{Bytes, BytesMut};
@@ -30,6 +30,7 @@ use num_traits::{AsPrimitive, Bounded, Zero};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize, Serializer};
use std::fmt::Debug;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
@@ -83,7 +84,6 @@ where
format!("binned_bytes_for_http BinnedRange::covering_range returned None"),
))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
//let _shape = entry.to_shape()?;
match PreBinnedPatchRange::covering_range(query.range().clone(), query.bin_count()) {
Ok(Some(pre_range)) => {
info!("binned_bytes_for_http found pre_range: {:?}", pre_range);
@@ -121,7 +121,7 @@ where
range: query.range().clone(),
agg_kind: query.agg_kind().clone(),
};
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
let ret = BinnedResponseStat {
stream: Box::pin(s),
@@ -150,10 +150,11 @@ fn make_num_pipeline_nty_end_evs_enp<PPP, NTY, END, EVS, ENP>(
where
PPP: PipelinePostProcessA,
PPP: PipelinePostProcessB<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>,
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Output> + 'static,
// TODO require these properties in general:
<ENP as EventsNodeProcessor>::Output: TimeBinnableType + PushableIndex + Appendable + 'static,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output:
TimeBinnableType<Output = <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output> + Unpin,
@@ -182,7 +183,7 @@ fn make_num_pipeline_nty_end<PPP, NTY, END>(
where
PPP: PipelinePostProcessA,
PPP: PipelinePostProcessB<MinMaxAvgBins<NTY>>,
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
{
match shape {
@@ -744,12 +745,23 @@ pub trait RangeOverlapInfo {
}
pub trait NumOps:
Sized + Copy + Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
Sized
+ Copy
+ Send
+ Unpin
+ Debug
+ Zero
+ AsPrimitive<f32>
+ Bounded
+ PartialOrd
+ SubFrId
+ Serialize
+ DeserializeOwned
{
}
impl<T> NumOps for T where
T: Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
T: Send + Unpin + Debug + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
{
}

4
disk/src/binned/pbv.rs
View File

@@ -6,7 +6,7 @@ use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, RangeCompletable
use crate::cache::{write_pb_cache_min_max_avg_scalar, CacheFileDesc, WrittenPbCache};
use crate::decode::{Endianness, EventValueFromBytes, EventValueShape, NumFromBytes};
use crate::frame::makeframe::FrameType;
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use crate::streamlog::Streamlog;
use crate::Sitemty;
@@ -123,7 +123,7 @@ where
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
Ok(Box::pin(ret))
}

87
disk/src/binned/query.rs
View File

@@ -54,7 +54,7 @@ impl PreBinnedQuery {
let disk_stats_every = disk_stats_every
.parse()
.map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
let ret = PreBinnedQuery {
let ret = Self {
patch: PreBinnedPatchCoord::new(bin_t_len, patch_t_len, patch_ix),
agg_kind: params
.get("aggKind")
@@ -177,8 +177,8 @@ pub struct BinnedQuery {
}
impl BinnedQuery {
pub fn new(channel: Channel, range: NanoRange, bin_count: u32, agg_kind: AggKind) -> BinnedQuery {
BinnedQuery {
pub fn new(channel: Channel, range: NanoRange, bin_count: u32, agg_kind: AggKind) -> Self {
Self {
channel,
range,
bin_count,
@@ -199,7 +199,7 @@ impl BinnedQuery {
let disk_stats_every = disk_stats_every
.parse()
.map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
let ret = BinnedQuery {
let ret = Self {
range: NanoRange {
beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
@@ -320,3 +320,82 @@ fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Err
};
Ok(ret)
}
// TODO move this query type out of this `binned` mod
#[derive(Clone, Debug)]
pub struct PlainEventsQuery {
channel: Channel,
range: NanoRange,
report_error: bool,
timeout: Duration,
}
impl PlainEventsQuery {
pub fn new(channel: Channel, range: NanoRange) -> Self {
Self {
channel,
range,
report_error: false,
timeout: Duration::from_millis(2000),
}
}
pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
let params = netpod::query_params(req.uri.query());
let beg_date = params.get("begDate").ok_or(Error::with_msg("missing begDate"))?;
let end_date = params.get("endDate").ok_or(Error::with_msg("missing endDate"))?;
let ret = Self {
range: NanoRange {
beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
},
channel: channel_from_params(&params)?,
report_error: params
.get("reportError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,
timeout: params
.get("timeout")
.map_or("2000", |k| k)
.parse::<u64>()
.map(|k| Duration::from_millis(k))
.map_err(|e| Error::with_msg(format!("can not parse timeout {:?}", e)))?,
};
Ok(ret)
}
pub fn range(&self) -> &NanoRange {
&self.range
}
pub fn channel(&self) -> &Channel {
&self.channel
}
pub fn report_error(&self) -> bool {
self.report_error
}
pub fn timeout(&self) -> Duration {
self.timeout
}
pub fn set_timeout(&mut self, k: Duration) {
self.timeout = k;
}
pub fn url(&self, host: &HostPort) -> String {
let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
format!(
"http://{}:{}/api/4/plain_events?channelBackend={}&channelName={}&begDate={}&endDate={}&timeout={}",
host.host,
host.port,
self.channel.backend,
self.channel.name,
Utc.timestamp_nanos(self.range.beg as i64).format(date_fmt),
Utc.timestamp_nanos(self.range.end as i64).format(date_fmt),
self.timeout.as_millis(),
)
}
}

162
disk/src/channelexec.rs Normal file
View File

@@ -0,0 +1,162 @@
use crate::agg::enp::Identity;
use crate::binned::NumOps;
use crate::decode::{
BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValuesDim0Case, EventValuesDim1Case,
LittleEndian, NumFromBytes,
};
use crate::frame::makeframe::Framable;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::{AggKind, ByteOrder, Channel, NanoRange, NodeConfigCached, PerfOpts, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use std::pin::Pin;
pub trait ChannelExecFunction {
type Output;
fn exec<NTY, END, EVS>(self, byte_order: END, event_value_shape: EVS) -> Result<Self::Output, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static;
}
fn channel_exec_nty_end_evs_enp<F, NTY, END, EVS>(
f: F,
byte_order: END,
event_value_shape: EVS,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
{
Ok(f.exec::<NTY, _, _>(byte_order, event_value_shape)?)
}
fn channel_exec_nty_end<F, NTY, END>(f: F, byte_order: END, shape: Shape) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
{
match shape {
Shape::Scalar => channel_exec_nty_end_evs_enp::<_, NTY, _, _>(f, byte_order, EventValuesDim0Case::new()),
Shape::Wave(n) => channel_exec_nty_end_evs_enp::<_, NTY, _, _>(f, byte_order, EventValuesDim1Case::new(n)),
}
}
macro_rules! match_end {
($f:expr, $nty:ident, $end:expr, $shape:expr, $node_config:expr) => {
match $end {
ByteOrder::LE => channel_exec_nty_end::<_, $nty, _>($f, LittleEndian {}, $shape),
ByteOrder::BE => channel_exec_nty_end::<_, $nty, _>($f, BigEndian {}, $shape),
}
};
}
fn channel_exec_config<F>(
f: F,
scalar_type: ScalarType,
byte_order: ByteOrder,
shape: Shape,
_node_config: &NodeConfigCached,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
{
match scalar_type {
ScalarType::U8 => match_end!(f, u8, byte_order, shape, node_config),
ScalarType::U16 => match_end!(f, u16, byte_order, shape, node_config),
ScalarType::U32 => match_end!(f, u32, byte_order, shape, node_config),
ScalarType::U64 => match_end!(f, u64, byte_order, shape, node_config),
ScalarType::I8 => match_end!(f, i8, byte_order, shape, node_config),
ScalarType::I16 => match_end!(f, i16, byte_order, shape, node_config),
ScalarType::I32 => match_end!(f, i32, byte_order, shape, node_config),
ScalarType::I64 => match_end!(f, i64, byte_order, shape, node_config),
ScalarType::F32 => match_end!(f, f32, byte_order, shape, node_config),
ScalarType::F64 => match_end!(f, f64, byte_order, shape, node_config),
}
}
pub async fn channel_exec<F>(
f: F,
channel: &Channel,
range: &NanoRange,
node_config: &NodeConfigCached,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
{
let channel_config = read_local_config(channel, &node_config.node).await?;
match extract_matching_config_entry(range, &channel_config)? {
MatchingConfigEntry::Multiple => Err(Error::with_msg("multiple config entries found"))?,
MatchingConfigEntry::None => {
// TODO function needs to provide some default.
err::todoval()
}
MatchingConfigEntry::Entry(entry) => {
let ret = channel_exec_config(
f,
entry.scalar_type.clone(),
entry.byte_order.clone(),
entry.to_shape()?,
node_config,
)?;
Ok(ret)
}
}
}
pub struct PlainEvents {
channel: Channel,
range: NanoRange,
agg_kind: AggKind,
node_config: NodeConfigCached,
}
impl PlainEvents {
pub fn new(channel: Channel, range: NanoRange, node_config: NodeConfigCached) -> Self {
Self {
channel,
range,
agg_kind: AggKind::DimXBins1,
node_config,
}
}
pub fn channel(&self) -> &Channel {
&self.channel
}
pub fn range(&self) -> &NanoRange {
&self.range
}
}
impl ChannelExecFunction for PlainEvents {
type Output = Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
fn exec<NTY, END, EVS>(self, byte_order: END, event_value_shape: EVS) -> Result<Self::Output, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
{
let _ = byte_order;
let _ = event_value_shape;
let perf_opts = PerfOpts { inmem_bufcap: 4096 };
let evq = EventsQuery {
channel: self.channel,
range: self.range,
agg_kind: self.agg_kind,
};
let s = MergedFromRemotes::<Identity<NTY>>::new(evq, perf_opts, self.node_config.node_config.cluster);
let s = s.map(|item| Box::new(item) as Box<dyn Framable>);
Ok(Box::pin(s))
}
}

1
disk/src/lib.rs
View File

@@ -24,6 +24,7 @@ pub mod binned;
pub mod binnedstream;
pub mod cache;
pub mod channelconfig;
pub mod channelexec;
pub mod dataopen;
pub mod decode;
pub mod eventblobs;

8
disk/src/merge/mergedfromremotes.rs
View File

@@ -15,7 +15,7 @@ use std::task::{Context, Poll};
type T001<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
type T002<T> = Pin<Box<dyn Future<Output = Result<T001<T>, Error>> + Send>>;
pub struct MergedFromRemotes2<ENP>
pub struct MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor,
{
@@ -26,7 +26,7 @@ where
errored: bool,
}
impl<ENP> MergedFromRemotes2<ENP>
impl<ENP> MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor + 'static,
<ENP as EventsNodeProcessor>::Output: 'static,
@@ -51,7 +51,7 @@ where
}
}
impl<ENP> Stream for MergedFromRemotes2<ENP>
impl<ENP> Stream for MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor + 'static,
<ENP as EventsNodeProcessor>::Output: PushableIndex + Appendable,
@@ -62,7 +62,7 @@ where
use Poll::*;
'outer: loop {
break if self.completed {
panic!("MergedFromRemotes poll_next on completed");
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
return Ready(None);