First basic plain event fetch as binary

This commit is contained in:
Dominik Werder
2021-06-10 17:37:40 +02:00
parent 99c45985ef
commit 0c43d5367e
9 changed files with 499 additions and 18 deletions

View File

@@ -20,6 +20,7 @@ use std::fmt::Debug;
use std::future::ready;
use tokio::io::AsyncRead;
pub mod events;
pub mod json;
#[test]

View File

@@ -0,0 +1,210 @@
use crate::nodes::require_test_hosts_running;
use chrono::{DateTime, Utc};
use disk::agg::streams::{StatsItem, StreamItem};
use disk::binned::query::PlainEventsQuery;
use disk::binned::{NumOps, RangeCompletableItem, WithLen};
use disk::decode::EventValues;
use disk::frame::inmem::InMemoryFrameAsyncReadStream;
use disk::frame::makeframe::FrameType;
use disk::streamlog::Streamlog;
use disk::Sitemty;
use err::Error;
use futures_util::{StreamExt, TryStreamExt};
use http::StatusCode;
use hyper::Body;
use netpod::log::*;
use netpod::{Channel, Cluster, HostPort, NanoRange, PerfOpts};
use std::fmt::Debug;
use std::future::ready;
use tokio::io::AsyncRead;
#[test]
fn get_plain_events_0() {
taskrun::run(get_plain_events_0_inner()).unwrap();
}
async fn get_plain_events_0_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
if true {
get_plain_events::<i32>(
"scalar-i32-be",
"1970-01-01T00:20:10.000Z",
"1970-01-01T00:20:50.000Z",
cluster,
true,
4,
)
.await?;
}
Ok(())
}
async fn get_plain_events<NTY>(
channel_name: &str,
beg_date: &str,
end_date: &str,
cluster: &Cluster,
_expect_range_complete: bool,
_expect_event_count: u64,
) -> Result<EventsResponse, Error>
where
NTY: NumOps,
{
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
let end_date: DateTime<Utc> = end_date.parse()?;
let channel_backend = "testbackend";
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let channel = Channel {
backend: channel_backend.into(),
name: channel_name.into(),
};
let range = NanoRange::from_date_time(beg_date, end_date);
let query = PlainEventsQuery::new(channel, range);
let hp = HostPort::from_node(node0);
let url = query.url(&hp);
info!("get_plain_events get {}", url);
let req = hyper::Request::builder()
.method(http::Method::GET)
.uri(url)
.header("accept", "application/octet-stream")
.body(Body::empty())?;
let client = hyper::Client::new();
let res = client.request(req).await?;
if res.status() != StatusCode::OK {
error!("client response {:?}", res);
}
let s1 = disk::cache::HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
let res = consume_plain_events::<NTY, _>(s2).await?;
let t2 = chrono::Utc::now();
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
info!("time {} ms", ms);
if !res.is_valid() {
Ok(res)
} else {
Ok(res)
}
}
#[derive(Debug)]
pub struct EventsResponse {
event_count: u64,
err_item_count: u64,
data_item_count: u64,
bytes_read: u64,
range_complete_count: u64,
log_item_count: u64,
stats_item_count: u64,
}
impl EventsResponse {
pub fn new() -> Self {
Self {
event_count: 0,
err_item_count: 0,
data_item_count: 0,
bytes_read: 0,
range_complete_count: 0,
log_item_count: 0,
stats_item_count: 0,
}
}
pub fn is_valid(&self) -> bool {
if self.range_complete_count > 1 {
false
} else {
true
}
}
}
async fn consume_plain_events<NTY, T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<EventsResponse, Error>
where
NTY: NumOps,
T: AsyncRead + Unpin,
{
let s1 = inp
.map_err(|e| error!("TEST GOT ERROR {:?}", e))
.filter_map(|item| {
let g = match item {
Ok(item) => match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
None
}
StreamItem::Stats(item) => {
info!("Stats: {:?}", item);
None
}
StreamItem::DataItem(frame) => {
if frame.tyid() != <Sitemty<EventValues<NTY>> as FrameType>::FRAME_TYPE_ID {
error!("test receives unexpected tyid {:x}", frame.tyid());
None
} else {
match bincode::deserialize::<Sitemty<EventValues<NTY>>>(frame.buf()) {
Ok(item) => match item {
Ok(item) => match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
Some(Ok(StreamItem::Log(item)))
}
item => {
info!("TEST GOT ITEM {:?}", item);
Some(Ok(item))
}
},
Err(e) => {
error!("TEST GOT ERROR FRAME: {:?}", e);
Some(Err(e))
}
},
Err(e) => {
error!("bincode error: {:?}", e);
Some(Err(e.into()))
}
}
}
}
},
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
};
ready(g)
})
.fold(EventsResponse::new(), |mut a, k| {
let g = match k {
Ok(StreamItem::Log(_item)) => {
a.log_item_count += 1;
a
}
Ok(StreamItem::Stats(item)) => match item {
StatsItem::EventDataReadStats(item) => {
a.bytes_read += item.parsed_bytes;
a
}
},
Ok(StreamItem::DataItem(item)) => match item {
RangeCompletableItem::RangeComplete => {
a.range_complete_count += 1;
a
}
RangeCompletableItem::Data(item) => {
a.data_item_count += 1;
a.event_count += WithLen::len(&item) as u64;
a
}
},
Err(_e) => {
a.err_item_count += 1;
a
}
};
ready(g)
});
let ret = s1.await;
info!("result: {:?}", ret);
Ok(ret)
}

View File

@@ -12,7 +12,7 @@ use crate::decode::{
LittleEndian, NumFromBytes,
};
use crate::frame::makeframe::{Framable, FrameType, SubFrId};
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use crate::Sitemty;
use bytes::{Bytes, BytesMut};
@@ -30,6 +30,7 @@ use num_traits::{AsPrimitive, Bounded, Zero};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize, Serializer};
use std::fmt::Debug;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
@@ -83,7 +84,6 @@ where
format!("binned_bytes_for_http BinnedRange::covering_range returned None"),
))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
//let _shape = entry.to_shape()?;
match PreBinnedPatchRange::covering_range(query.range().clone(), query.bin_count()) {
Ok(Some(pre_range)) => {
info!("binned_bytes_for_http found pre_range: {:?}", pre_range);
@@ -121,7 +121,7 @@ where
range: query.range().clone(),
agg_kind: query.agg_kind().clone(),
};
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, node_config.node_config.cluster.clone());
let s = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
let ret = BinnedResponseStat {
stream: Box::pin(s),
@@ -150,10 +150,11 @@ fn make_num_pipeline_nty_end_evs_enp<PPP, NTY, END, EVS, ENP>(
where
PPP: PipelinePostProcessA,
PPP: PipelinePostProcessB<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>,
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Output> + 'static,
// TODO require these properties in general:
<ENP as EventsNodeProcessor>::Output: TimeBinnableType + PushableIndex + Appendable + 'static,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output:
TimeBinnableType<Output = <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output> + Unpin,
@@ -182,7 +183,7 @@ fn make_num_pipeline_nty_end<PPP, NTY, END>(
where
PPP: PipelinePostProcessA,
PPP: PipelinePostProcessB<MinMaxAvgBins<NTY>>,
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
{
match shape {
@@ -744,12 +745,23 @@ pub trait RangeOverlapInfo {
}
pub trait NumOps:
Sized + Copy + Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
Sized
+ Copy
+ Send
+ Unpin
+ Debug
+ Zero
+ AsPrimitive<f32>
+ Bounded
+ PartialOrd
+ SubFrId
+ Serialize
+ DeserializeOwned
{
}
impl<T> NumOps for T where
T: Send + Unpin + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
T: Send + Unpin + Debug + Zero + AsPrimitive<f32> + Bounded + PartialOrd + SubFrId + Serialize + DeserializeOwned
{
}

View File

@@ -6,7 +6,7 @@ use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, RangeCompletable
use crate::cache::{write_pb_cache_min_max_avg_scalar, CacheFileDesc, WrittenPbCache};
use crate::decode::{Endianness, EventValueFromBytes, EventValueShape, NumFromBytes};
use crate::frame::makeframe::FrameType;
use crate::merge::mergedfromremotes::MergedFromRemotes2;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use crate::streamlog::Streamlog;
use crate::Sitemty;
@@ -123,7 +123,7 @@ where
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes2::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
let ret = TBinnerStream::<_, <ENP as EventsNodeProcessor>::Output>::new(s, range);
Ok(Box::pin(ret))
}

View File

@@ -54,7 +54,7 @@ impl PreBinnedQuery {
let disk_stats_every = disk_stats_every
.parse()
.map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
let ret = PreBinnedQuery {
let ret = Self {
patch: PreBinnedPatchCoord::new(bin_t_len, patch_t_len, patch_ix),
agg_kind: params
.get("aggKind")
@@ -177,8 +177,8 @@ pub struct BinnedQuery {
}
impl BinnedQuery {
pub fn new(channel: Channel, range: NanoRange, bin_count: u32, agg_kind: AggKind) -> BinnedQuery {
BinnedQuery {
pub fn new(channel: Channel, range: NanoRange, bin_count: u32, agg_kind: AggKind) -> Self {
Self {
channel,
range,
bin_count,
@@ -199,7 +199,7 @@ impl BinnedQuery {
let disk_stats_every = disk_stats_every
.parse()
.map_err(|e| Error::with_msg(format!("can not parse diskStatsEveryKb {:?}", e)))?;
let ret = BinnedQuery {
let ret = Self {
range: NanoRange {
beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
@@ -320,3 +320,82 @@ fn channel_from_params(params: &BTreeMap<String, String>) -> Result<Channel, Err
};
Ok(ret)
}
// TODO move this query type out of this `binned` mod
#[derive(Clone, Debug)]
pub struct PlainEventsQuery {
channel: Channel,
range: NanoRange,
report_error: bool,
timeout: Duration,
}
impl PlainEventsQuery {
pub fn new(channel: Channel, range: NanoRange) -> Self {
Self {
channel,
range,
report_error: false,
timeout: Duration::from_millis(2000),
}
}
pub fn from_request(req: &http::request::Parts) -> Result<Self, Error> {
let params = netpod::query_params(req.uri.query());
let beg_date = params.get("begDate").ok_or(Error::with_msg("missing begDate"))?;
let end_date = params.get("endDate").ok_or(Error::with_msg("missing endDate"))?;
let ret = Self {
range: NanoRange {
beg: beg_date.parse::<DateTime<Utc>>()?.to_nanos(),
end: end_date.parse::<DateTime<Utc>>()?.to_nanos(),
},
channel: channel_from_params(&params)?,
report_error: params
.get("reportError")
.map_or("false", |k| k)
.parse()
.map_err(|e| Error::with_msg(format!("can not parse reportError {:?}", e)))?,
timeout: params
.get("timeout")
.map_or("2000", |k| k)
.parse::<u64>()
.map(|k| Duration::from_millis(k))
.map_err(|e| Error::with_msg(format!("can not parse timeout {:?}", e)))?,
};
Ok(ret)
}
pub fn range(&self) -> &NanoRange {
&self.range
}
pub fn channel(&self) -> &Channel {
&self.channel
}
pub fn report_error(&self) -> bool {
self.report_error
}
pub fn timeout(&self) -> Duration {
self.timeout
}
pub fn set_timeout(&mut self, k: Duration) {
self.timeout = k;
}
pub fn url(&self, host: &HostPort) -> String {
let date_fmt = "%Y-%m-%dT%H:%M:%S.%3fZ";
format!(
"http://{}:{}/api/4/plain_events?channelBackend={}&channelName={}&begDate={}&endDate={}&timeout={}",
host.host,
host.port,
self.channel.backend,
self.channel.name,
Utc.timestamp_nanos(self.range.beg as i64).format(date_fmt),
Utc.timestamp_nanos(self.range.end as i64).format(date_fmt),
self.timeout.as_millis(),
)
}
}

162
disk/src/channelexec.rs Normal file
View File

@@ -0,0 +1,162 @@
use crate::agg::enp::Identity;
use crate::binned::NumOps;
use crate::decode::{
BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValuesDim0Case, EventValuesDim1Case,
LittleEndian, NumFromBytes,
};
use crate::frame::makeframe::Framable;
use crate::merge::mergedfromremotes::MergedFromRemotes;
use crate::raw::EventsQuery;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::{AggKind, ByteOrder, Channel, NanoRange, NodeConfigCached, PerfOpts, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use std::pin::Pin;
pub trait ChannelExecFunction {
type Output;
fn exec<NTY, END, EVS>(self, byte_order: END, event_value_shape: EVS) -> Result<Self::Output, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static;
}
fn channel_exec_nty_end_evs_enp<F, NTY, END, EVS>(
f: F,
byte_order: END,
event_value_shape: EVS,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
{
Ok(f.exec::<NTY, _, _>(byte_order, event_value_shape)?)
}
fn channel_exec_nty_end<F, NTY, END>(f: F, byte_order: END, shape: Shape) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
{
match shape {
Shape::Scalar => channel_exec_nty_end_evs_enp::<_, NTY, _, _>(f, byte_order, EventValuesDim0Case::new()),
Shape::Wave(n) => channel_exec_nty_end_evs_enp::<_, NTY, _, _>(f, byte_order, EventValuesDim1Case::new(n)),
}
}
macro_rules! match_end {
($f:expr, $nty:ident, $end:expr, $shape:expr, $node_config:expr) => {
match $end {
ByteOrder::LE => channel_exec_nty_end::<_, $nty, _>($f, LittleEndian {}, $shape),
ByteOrder::BE => channel_exec_nty_end::<_, $nty, _>($f, BigEndian {}, $shape),
}
};
}
fn channel_exec_config<F>(
f: F,
scalar_type: ScalarType,
byte_order: ByteOrder,
shape: Shape,
_node_config: &NodeConfigCached,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
{
match scalar_type {
ScalarType::U8 => match_end!(f, u8, byte_order, shape, node_config),
ScalarType::U16 => match_end!(f, u16, byte_order, shape, node_config),
ScalarType::U32 => match_end!(f, u32, byte_order, shape, node_config),
ScalarType::U64 => match_end!(f, u64, byte_order, shape, node_config),
ScalarType::I8 => match_end!(f, i8, byte_order, shape, node_config),
ScalarType::I16 => match_end!(f, i16, byte_order, shape, node_config),
ScalarType::I32 => match_end!(f, i32, byte_order, shape, node_config),
ScalarType::I64 => match_end!(f, i64, byte_order, shape, node_config),
ScalarType::F32 => match_end!(f, f32, byte_order, shape, node_config),
ScalarType::F64 => match_end!(f, f64, byte_order, shape, node_config),
}
}
pub async fn channel_exec<F>(
f: F,
channel: &Channel,
range: &NanoRange,
node_config: &NodeConfigCached,
) -> Result<F::Output, Error>
where
F: ChannelExecFunction,
{
let channel_config = read_local_config(channel, &node_config.node).await?;
match extract_matching_config_entry(range, &channel_config)? {
MatchingConfigEntry::Multiple => Err(Error::with_msg("multiple config entries found"))?,
MatchingConfigEntry::None => {
// TODO function needs to provide some default.
err::todoval()
}
MatchingConfigEntry::Entry(entry) => {
let ret = channel_exec_config(
f,
entry.scalar_type.clone(),
entry.byte_order.clone(),
entry.to_shape()?,
node_config,
)?;
Ok(ret)
}
}
}
pub struct PlainEvents {
channel: Channel,
range: NanoRange,
agg_kind: AggKind,
node_config: NodeConfigCached,
}
impl PlainEvents {
pub fn new(channel: Channel, range: NanoRange, node_config: NodeConfigCached) -> Self {
Self {
channel,
range,
agg_kind: AggKind::DimXBins1,
node_config,
}
}
pub fn channel(&self) -> &Channel {
&self.channel
}
pub fn range(&self) -> &NanoRange {
&self.range
}
}
impl ChannelExecFunction for PlainEvents {
type Output = Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
fn exec<NTY, END, EVS>(self, byte_order: END, event_value_shape: EVS) -> Result<Self::Output, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
{
let _ = byte_order;
let _ = event_value_shape;
let perf_opts = PerfOpts { inmem_bufcap: 4096 };
let evq = EventsQuery {
channel: self.channel,
range: self.range,
agg_kind: self.agg_kind,
};
let s = MergedFromRemotes::<Identity<NTY>>::new(evq, perf_opts, self.node_config.node_config.cluster);
let s = s.map(|item| Box::new(item) as Box<dyn Framable>);
Ok(Box::pin(s))
}
}

View File

@@ -24,6 +24,7 @@ pub mod binned;
pub mod binnedstream;
pub mod cache;
pub mod channelconfig;
pub mod channelexec;
pub mod dataopen;
pub mod decode;
pub mod eventblobs;

View File

@@ -15,7 +15,7 @@ use std::task::{Context, Poll};
type T001<T> = Pin<Box<dyn Stream<Item = Sitemty<T>> + Send>>;
type T002<T> = Pin<Box<dyn Future<Output = Result<T001<T>, Error>> + Send>>;
pub struct MergedFromRemotes2<ENP>
pub struct MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor,
{
@@ -26,7 +26,7 @@ where
errored: bool,
}
impl<ENP> MergedFromRemotes2<ENP>
impl<ENP> MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor + 'static,
<ENP as EventsNodeProcessor>::Output: 'static,
@@ -51,7 +51,7 @@ where
}
}
impl<ENP> Stream for MergedFromRemotes2<ENP>
impl<ENP> Stream for MergedFromRemotes<ENP>
where
ENP: EventsNodeProcessor + 'static,
<ENP as EventsNodeProcessor>::Output: PushableIndex + Appendable,
@@ -62,7 +62,7 @@ where
use Poll::*;
'outer: loop {
break if self.completed {
panic!("MergedFromRemotes poll_next on completed");
panic!("poll_next on completed");
} else if self.errored {
self.completed = true;
return Ready(None);

View File

@@ -1,7 +1,7 @@
use crate::gather::gather_get_json;
use bytes::Bytes;
use disk::binned::prebinned::pre_binned_bytes_for_http;
use disk::binned::query::{BinnedQuery, PreBinnedQuery};
use disk::binned::query::{BinnedQuery, PlainEventsQuery, PreBinnedQuery};
use disk::raw::conn::events_service;
use err::Error;
use future::Future;
@@ -169,6 +169,12 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/4/plain_events" {
if req.method() == Method::GET {
Ok(plain_events(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path.starts_with("/api/4/gather/") {
if req.method() == Method::GET {
Ok(gather_get_json(req, &node_config).await?)
@@ -386,6 +392,16 @@ async fn prebinned(req: Request<Body>, node_config: &NodeConfigCached) -> Result
Ok(ret)
}
async fn plain_events(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
let query = PlainEventsQuery::from_request(&head)?;
let op = disk::channelexec::PlainEvents::new(query.channel().clone(), query.range().clone(), node_config.clone());
let s = disk::channelexec::channel_exec(op, query.channel(), query.range(), node_config).await?;
let s = s.map(|item| item.make_frame());
let ret = response(StatusCode::OK).body(BodyStream::wrapped(s, format!("plain_events")))?;
Ok(ret)
}
#[derive(Debug, Serialize, Deserialize)]
pub struct NodeStatus {
database_size: u64,