Iterate through pb files and parse the header
This commit is contained in:
Dominik Werder
256
daqbufp2/src/test/binnedbinary.rs
Normal file
256
daqbufp2/src/test/binnedbinary.rs
Normal file
@@ -0,0 +1,256 @@
|
||||
use crate::nodes::require_test_hosts_running;
|
||||
use chrono::{DateTime, Utc};
|
||||
use disk::agg::streams::{StatsItem, StreamItem};
|
||||
use disk::binned::query::{BinnedQuery, CacheUsage};
|
||||
use disk::binned::{MinMaxAvgBins, RangeCompletableItem, WithLen};
|
||||
use disk::frame::inmem::InMemoryFrameAsyncReadStream;
|
||||
use disk::frame::makeframe::{FrameType, SubFrId};
|
||||
use disk::streamlog::Streamlog;
|
||||
use disk::Sitemty;
|
||||
use err::Error;
|
||||
use futures_util::{StreamExt, TryStreamExt};
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::{AggKind, AppendToUrl, Channel, Cluster, HostPort, NanoRange, PerfOpts, APP_OCTET};
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::fmt;
|
||||
use std::future::ready;
|
||||
use tokio::io::AsyncRead;
|
||||
use url::Url;
|
||||
|
||||
#[test]
|
||||
fn get_binned_binary() {
|
||||
taskrun::run(get_binned_binary_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_binned_binary_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
if true {
|
||||
get_binned_channel::<i32>(
|
||||
"scalar-i32-be",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:50.000Z",
|
||||
3,
|
||||
cluster,
|
||||
true,
|
||||
4,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if true {
|
||||
get_binned_channel::<f64>(
|
||||
"wave-f64-be-n21",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:30.000Z",
|
||||
2,
|
||||
cluster,
|
||||
true,
|
||||
2,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if true {
|
||||
get_binned_channel::<u16>(
|
||||
"wave-u16-le-n77",
|
||||
"1970-01-01T01:11:00.000Z",
|
||||
"1970-01-01T01:35:00.000Z",
|
||||
7,
|
||||
cluster,
|
||||
true,
|
||||
24,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if true {
|
||||
get_binned_channel::<u16>(
|
||||
"wave-u16-le-n77",
|
||||
"1970-01-01T01:42:00.000Z",
|
||||
"1970-01-01T03:55:00.000Z",
|
||||
2,
|
||||
cluster,
|
||||
true,
|
||||
3,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_binned_channel<NTY>(
|
||||
channel_name: &str,
|
||||
beg_date: &str,
|
||||
end_date: &str,
|
||||
bin_count: u32,
|
||||
cluster: &Cluster,
|
||||
expect_range_complete: bool,
|
||||
expect_bin_count: u64,
|
||||
) -> Result<BinnedResponse, Error>
|
||||
where
|
||||
NTY: fmt::Debug + SubFrId + DeserializeOwned,
|
||||
{
|
||||
let t1 = Utc::now();
|
||||
let agg_kind = AggKind::DimXBins1;
|
||||
let node0 = &cluster.nodes[0];
|
||||
let beg_date: DateTime<Utc> = beg_date.parse()?;
|
||||
let end_date: DateTime<Utc> = end_date.parse()?;
|
||||
let channel_backend = "testbackend";
|
||||
let perf_opts = PerfOpts { inmem_bufcap: 512 };
|
||||
let channel = Channel {
|
||||
backend: channel_backend.into(),
|
||||
name: channel_name.into(),
|
||||
};
|
||||
let range = NanoRange::from_date_time(beg_date, end_date);
|
||||
let mut query = BinnedQuery::new(channel, range, bin_count, agg_kind);
|
||||
query.set_cache_usage(CacheUsage::Ignore);
|
||||
query.set_disk_io_buffer_size(1024 * 16);
|
||||
let hp = HostPort::from_node(node0);
|
||||
let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", hp.host, hp.port))?;
|
||||
query.append_to_url(&mut url);
|
||||
let url = url;
|
||||
info!("get_binned_channel get {}", url);
|
||||
let req = hyper::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(url.to_string())
|
||||
.header(http::header::ACCEPT, APP_OCTET)
|
||||
.body(Body::empty())?;
|
||||
let client = hyper::Client::new();
|
||||
let res = client.request(req).await?;
|
||||
if res.status() != StatusCode::OK {
|
||||
error!("client response {:?}", res);
|
||||
}
|
||||
let s1 = disk::cache::HttpBodyAsAsyncRead::new(res);
|
||||
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
|
||||
let res = consume_binned_response::<NTY, _>(s2).await?;
|
||||
let t2 = chrono::Utc::now();
|
||||
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
|
||||
info!("get_cached_0 DONE bin_count {} time {} ms", res.bin_count, ms);
|
||||
if !res.is_valid() {
|
||||
Err(Error::with_msg(format!("invalid response: {:?}", res)))
|
||||
} else if res.range_complete_count == 0 && expect_range_complete {
|
||||
Err(Error::with_msg(format!("expect range complete: {:?}", res)))
|
||||
} else if res.bin_count != expect_bin_count {
|
||||
Err(Error::with_msg(format!("bin count mismatch: {:?}", res)))
|
||||
} else {
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct BinnedResponse {
|
||||
bin_count: u64,
|
||||
err_item_count: u64,
|
||||
data_item_count: u64,
|
||||
bytes_read: u64,
|
||||
range_complete_count: u64,
|
||||
log_item_count: u64,
|
||||
stats_item_count: u64,
|
||||
}
|
||||
|
||||
impl BinnedResponse {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bin_count: 0,
|
||||
err_item_count: 0,
|
||||
data_item_count: 0,
|
||||
bytes_read: 0,
|
||||
range_complete_count: 0,
|
||||
log_item_count: 0,
|
||||
stats_item_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_valid(&self) -> bool {
|
||||
if self.range_complete_count > 1 {
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn consume_binned_response<NTY, T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<BinnedResponse, Error>
|
||||
where
|
||||
NTY: fmt::Debug + SubFrId + DeserializeOwned,
|
||||
T: AsyncRead + Unpin,
|
||||
{
|
||||
let s1 = inp
|
||||
.map_err(|e| error!("TEST GOT ERROR {:?}", e))
|
||||
.filter_map(|item| {
|
||||
let g = match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::Log(item) => {
|
||||
Streamlog::emit(&item);
|
||||
None
|
||||
}
|
||||
StreamItem::Stats(item) => {
|
||||
info!("Stats: {:?}", item);
|
||||
None
|
||||
}
|
||||
StreamItem::DataItem(frame) => {
|
||||
if frame.tyid() != <Sitemty<MinMaxAvgBins<NTY>> as FrameType>::FRAME_TYPE_ID {
|
||||
error!("test receives unexpected tyid {:x}", frame.tyid());
|
||||
}
|
||||
match bincode::deserialize::<Sitemty<MinMaxAvgBins<NTY>>>(frame.buf()) {
|
||||
Ok(item) => match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::Log(item) => {
|
||||
Streamlog::emit(&item);
|
||||
Some(Ok(StreamItem::Log(item)))
|
||||
}
|
||||
item => {
|
||||
info!("TEST GOT ITEM {:?}", item);
|
||||
Some(Ok(item))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("TEST GOT ERROR FRAME: {:?}", e);
|
||||
Some(Err(e))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("bincode error: {:?}", e);
|
||||
Some(Err(e.into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
|
||||
};
|
||||
ready(g)
|
||||
})
|
||||
.fold(BinnedResponse::new(), |mut a, k| {
|
||||
let g = match k {
|
||||
Ok(StreamItem::Log(_item)) => {
|
||||
a.log_item_count += 1;
|
||||
a
|
||||
}
|
||||
Ok(StreamItem::Stats(item)) => match item {
|
||||
StatsItem::EventDataReadStats(item) => {
|
||||
a.bytes_read += item.parsed_bytes;
|
||||
a
|
||||
}
|
||||
},
|
||||
Ok(StreamItem::DataItem(item)) => match item {
|
||||
RangeCompletableItem::RangeComplete => {
|
||||
a.range_complete_count += 1;
|
||||
a
|
||||
}
|
||||
RangeCompletableItem::Data(item) => {
|
||||
a.data_item_count += 1;
|
||||
a.bin_count += WithLen::len(&item) as u64;
|
||||
a
|
||||
}
|
||||
},
|
||||
Err(_e) => {
|
||||
a.err_item_count += 1;
|
||||
a
|
||||
}
|
||||
};
|
||||
ready(g)
|
||||
});
|
||||
let ret = s1.await;
|
||||
info!("BinnedResponse: {:?}", ret);
|
||||
Ok(ret)
|
||||
}
|
||||
151
daqbufp2/src/test/binnedjson.rs
Normal file
151
daqbufp2/src/test/binnedjson.rs
Normal file
@@ -0,0 +1,151 @@
|
||||
use crate::nodes::require_test_hosts_running;
|
||||
use chrono::{DateTime, Utc};
|
||||
use disk::binned::query::{BinnedQuery, CacheUsage};
|
||||
use err::Error;
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::{AggKind, AppendToUrl, Channel, Cluster, NanoRange, APP_JSON};
|
||||
use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
#[test]
|
||||
fn get_binned_json_0() {
|
||||
taskrun::run(get_binned_json_0_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_binned_json_0_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
get_binned_json_common(
|
||||
"scalar-i32-be",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T01:20:30.000Z",
|
||||
10,
|
||||
AggKind::DimXBins1,
|
||||
cluster,
|
||||
13,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_binned_json_1() {
|
||||
taskrun::run(get_binned_json_1_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_binned_json_1_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
get_binned_json_common(
|
||||
"wave-f64-be-n21",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T01:20:45.000Z",
|
||||
10,
|
||||
AggKind::DimXBins1,
|
||||
cluster,
|
||||
13,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_binned_json_2() {
|
||||
taskrun::run(get_binned_json_2_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_binned_json_2_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
get_binned_json_common(
|
||||
"wave-f64-be-n21",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:20.000Z",
|
||||
2,
|
||||
AggKind::DimXBinsN(3),
|
||||
cluster,
|
||||
2,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_binned_json_common(
|
||||
channel_name: &str,
|
||||
beg_date: &str,
|
||||
end_date: &str,
|
||||
bin_count: u32,
|
||||
agg_kind: AggKind,
|
||||
cluster: &Cluster,
|
||||
expect_bin_count: u32,
|
||||
expect_finalised_range: bool,
|
||||
) -> Result<(), Error> {
|
||||
let t1 = Utc::now();
|
||||
let node0 = &cluster.nodes[0];
|
||||
let beg_date: DateTime<Utc> = beg_date.parse()?;
|
||||
let end_date: DateTime<Utc> = end_date.parse()?;
|
||||
let channel_backend = "testbackend";
|
||||
let channel = Channel {
|
||||
backend: channel_backend.into(),
|
||||
name: channel_name.into(),
|
||||
};
|
||||
let range = NanoRange::from_date_time(beg_date, end_date);
|
||||
let mut query = BinnedQuery::new(channel, range, bin_count, agg_kind);
|
||||
query.set_timeout(Duration::from_millis(15000));
|
||||
query.set_cache_usage(CacheUsage::Ignore);
|
||||
let mut url = Url::parse(&format!("http://{}:{}/api/4/binned", node0.host, node0.port))?;
|
||||
query.append_to_url(&mut url);
|
||||
let url = url;
|
||||
info!("get_binned_json_common get {}", url);
|
||||
let req = hyper::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(url.to_string())
|
||||
.header(http::header::ACCEPT, APP_JSON)
|
||||
.body(Body::empty())?;
|
||||
let client = hyper::Client::new();
|
||||
let res = client.request(req).await?;
|
||||
if res.status() != StatusCode::OK {
|
||||
error!("get_binned_json_common client response {:?}", res);
|
||||
}
|
||||
let res = hyper::body::to_bytes(res.into_body()).await?;
|
||||
let t2 = chrono::Utc::now();
|
||||
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
|
||||
info!("get_binned_json_common DONE time {} ms", ms);
|
||||
let res = String::from_utf8_lossy(&res).to_string();
|
||||
//info!("get_binned_json_common res: {}", res);
|
||||
let res: serde_json::Value = serde_json::from_str(res.as_str())?;
|
||||
info!(
|
||||
"result from endpoint: --------------\n{}\n--------------",
|
||||
serde_json::to_string_pretty(&res)?
|
||||
);
|
||||
// TODO enable in future:
|
||||
if false {
|
||||
if expect_finalised_range {
|
||||
if !res
|
||||
.get("finalisedRange")
|
||||
.ok_or(Error::with_msg("missing finalisedRange"))?
|
||||
.as_bool()
|
||||
.ok_or(Error::with_msg("key finalisedRange not bool"))?
|
||||
{
|
||||
return Err(Error::with_msg("expected finalisedRange"));
|
||||
}
|
||||
} else if res.get("finalisedRange").is_some() {
|
||||
return Err(Error::with_msg("expect absent finalisedRange"));
|
||||
}
|
||||
}
|
||||
if res.get("counts").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
|
||||
return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
|
||||
}
|
||||
if res.get("mins").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
|
||||
return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
|
||||
}
|
||||
if res.get("maxs").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
|
||||
return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
|
||||
}
|
||||
if res.get("avgs").unwrap().as_array().unwrap().len() != expect_bin_count as usize {
|
||||
return Err(Error::with_msg(format!("expect_bin_count {}", expect_bin_count)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
298
daqbufp2/src/test/events.rs
Normal file
298
daqbufp2/src/test/events.rs
Normal file
@@ -0,0 +1,298 @@
|
||||
use crate::nodes::require_test_hosts_running;
|
||||
use chrono::{DateTime, Utc};
|
||||
use disk::agg::streams::{StatsItem, StreamItem};
|
||||
use disk::binned::{NumOps, RangeCompletableItem, WithLen};
|
||||
use disk::decode::EventValues;
|
||||
use disk::events::{PlainEventsBinaryQuery, PlainEventsJsonQuery};
|
||||
use disk::frame::inmem::InMemoryFrameAsyncReadStream;
|
||||
use disk::frame::makeframe::FrameType;
|
||||
use disk::streamlog::Streamlog;
|
||||
use disk::Sitemty;
|
||||
use err::Error;
|
||||
use futures_util::{StreamExt, TryStreamExt};
|
||||
use http::StatusCode;
|
||||
use hyper::Body;
|
||||
use netpod::log::*;
|
||||
use netpod::{AppendToUrl, Channel, Cluster, HostPort, NanoRange, PerfOpts, APP_JSON, APP_OCTET};
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::fmt::Debug;
|
||||
use std::future::ready;
|
||||
use tokio::io::AsyncRead;
|
||||
use url::Url;
|
||||
|
||||
#[test]
|
||||
fn get_plain_events_binary_0() {
|
||||
taskrun::run(get_plain_events_binary_0_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_plain_events_binary_0_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
if true {
|
||||
get_plain_events_binary::<i32>(
|
||||
"scalar-i32-be",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:50.000Z",
|
||||
cluster,
|
||||
true,
|
||||
4,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_plain_events_binary<NTY>(
|
||||
channel_name: &str,
|
||||
beg_date: &str,
|
||||
end_date: &str,
|
||||
cluster: &Cluster,
|
||||
_expect_range_complete: bool,
|
||||
_expect_event_count: u64,
|
||||
) -> Result<EventsResponse, Error>
|
||||
where
|
||||
NTY: NumOps,
|
||||
{
|
||||
let t1 = Utc::now();
|
||||
let node0 = &cluster.nodes[0];
|
||||
let beg_date: DateTime<Utc> = beg_date.parse()?;
|
||||
let end_date: DateTime<Utc> = end_date.parse()?;
|
||||
let channel_backend = "testbackend";
|
||||
let perf_opts = PerfOpts { inmem_bufcap: 512 };
|
||||
let channel = Channel {
|
||||
backend: channel_backend.into(),
|
||||
name: channel_name.into(),
|
||||
};
|
||||
let range = NanoRange::from_date_time(beg_date, end_date);
|
||||
let query = PlainEventsBinaryQuery::new(channel, range, 1024 * 4);
|
||||
let hp = HostPort::from_node(node0);
|
||||
let mut url = Url::parse(&format!("http://{}:{}", hp.host, hp.port))?;
|
||||
query.append_to_url(&mut url);
|
||||
let url = url;
|
||||
info!("get_plain_events get {}", url);
|
||||
let req = hyper::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(url.to_string())
|
||||
.header(http::header::ACCEPT, APP_OCTET)
|
||||
.body(Body::empty())?;
|
||||
let client = hyper::Client::new();
|
||||
let res = client.request(req).await?;
|
||||
if res.status() != StatusCode::OK {
|
||||
error!("client response {:?}", res);
|
||||
}
|
||||
let s1 = disk::cache::HttpBodyAsAsyncRead::new(res);
|
||||
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
|
||||
let res = consume_plain_events_binary::<NTY, _>(s2).await?;
|
||||
let t2 = chrono::Utc::now();
|
||||
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
|
||||
info!("time {} ms", ms);
|
||||
if !res.is_valid() {
|
||||
Ok(res)
|
||||
} else {
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct EventsResponse {
|
||||
event_count: u64,
|
||||
err_item_count: u64,
|
||||
data_item_count: u64,
|
||||
bytes_read: u64,
|
||||
range_complete_count: u64,
|
||||
log_item_count: u64,
|
||||
stats_item_count: u64,
|
||||
}
|
||||
|
||||
impl EventsResponse {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
event_count: 0,
|
||||
err_item_count: 0,
|
||||
data_item_count: 0,
|
||||
bytes_read: 0,
|
||||
range_complete_count: 0,
|
||||
log_item_count: 0,
|
||||
stats_item_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_valid(&self) -> bool {
|
||||
if self.range_complete_count > 1 {
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn consume_plain_events_binary<NTY, T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<EventsResponse, Error>
|
||||
where
|
||||
NTY: NumOps,
|
||||
T: AsyncRead + Unpin,
|
||||
{
|
||||
let s1 = inp
|
||||
.map_err(|e| error!("TEST GOT ERROR {:?}", e))
|
||||
.filter_map(|item| {
|
||||
let g = match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::Log(item) => {
|
||||
Streamlog::emit(&item);
|
||||
None
|
||||
}
|
||||
StreamItem::Stats(item) => {
|
||||
info!("Stats: {:?}", item);
|
||||
None
|
||||
}
|
||||
StreamItem::DataItem(frame) => {
|
||||
if frame.tyid() != <Sitemty<EventValues<NTY>> as FrameType>::FRAME_TYPE_ID {
|
||||
error!("test receives unexpected tyid {:x}", frame.tyid());
|
||||
None
|
||||
} else {
|
||||
match bincode::deserialize::<Sitemty<EventValues<NTY>>>(frame.buf()) {
|
||||
Ok(item) => match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::Log(item) => {
|
||||
Streamlog::emit(&item);
|
||||
Some(Ok(StreamItem::Log(item)))
|
||||
}
|
||||
item => {
|
||||
info!("TEST GOT ITEM {:?}", item);
|
||||
Some(Ok(item))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("TEST GOT ERROR FRAME: {:?}", e);
|
||||
Some(Err(e))
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("bincode error: {:?}", e);
|
||||
Some(Err(e.into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
|
||||
};
|
||||
ready(g)
|
||||
})
|
||||
.fold(EventsResponse::new(), |mut a, k| {
|
||||
let g = match k {
|
||||
Ok(StreamItem::Log(_item)) => {
|
||||
a.log_item_count += 1;
|
||||
a
|
||||
}
|
||||
Ok(StreamItem::Stats(item)) => match item {
|
||||
StatsItem::EventDataReadStats(item) => {
|
||||
a.bytes_read += item.parsed_bytes;
|
||||
a
|
||||
}
|
||||
},
|
||||
Ok(StreamItem::DataItem(item)) => match item {
|
||||
RangeCompletableItem::RangeComplete => {
|
||||
a.range_complete_count += 1;
|
||||
a
|
||||
}
|
||||
RangeCompletableItem::Data(item) => {
|
||||
a.data_item_count += 1;
|
||||
a.event_count += WithLen::len(&item) as u64;
|
||||
a
|
||||
}
|
||||
},
|
||||
Err(_e) => {
|
||||
a.err_item_count += 1;
|
||||
a
|
||||
}
|
||||
};
|
||||
ready(g)
|
||||
});
|
||||
let ret = s1.await;
|
||||
info!("result: {:?}", ret);
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_plain_events_json_0() {
|
||||
taskrun::run(get_plain_events_json_0_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_plain_events_json_0_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
get_plain_events_json(
|
||||
"scalar-i32-be",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:12.000Z",
|
||||
cluster,
|
||||
true,
|
||||
4,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_plain_events_json_1() {
|
||||
taskrun::run(get_plain_events_json_1_inner()).unwrap();
|
||||
}
|
||||
|
||||
async fn get_plain_events_json_1_inner() -> Result<(), Error> {
|
||||
let rh = require_test_hosts_running()?;
|
||||
let cluster = &rh.cluster;
|
||||
get_plain_events_json(
|
||||
"wave-f64-be-n21",
|
||||
"1970-01-01T00:20:10.000Z",
|
||||
"1970-01-01T00:20:12.000Z",
|
||||
cluster,
|
||||
true,
|
||||
4,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_plain_events_json(
|
||||
channel_name: &str,
|
||||
beg_date: &str,
|
||||
end_date: &str,
|
||||
cluster: &Cluster,
|
||||
_expect_range_complete: bool,
|
||||
_expect_event_count: u64,
|
||||
) -> Result<(), Error> {
|
||||
let t1 = Utc::now();
|
||||
let node0 = &cluster.nodes[0];
|
||||
let beg_date: DateTime<Utc> = beg_date.parse()?;
|
||||
let end_date: DateTime<Utc> = end_date.parse()?;
|
||||
let channel_backend = "testbackend";
|
||||
let channel = Channel {
|
||||
backend: channel_backend.into(),
|
||||
name: channel_name.into(),
|
||||
};
|
||||
let range = NanoRange::from_date_time(beg_date, end_date);
|
||||
let query = PlainEventsJsonQuery::new(channel, range, 1024 * 4, false);
|
||||
let hp = HostPort::from_node(node0);
|
||||
let mut url = Url::parse(&format!("http://{}:{}/api/4/events", hp.host, hp.port))?;
|
||||
query.append_to_url(&mut url);
|
||||
let url = url;
|
||||
info!("get_plain_events get {}", url);
|
||||
let req = hyper::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(url.to_string())
|
||||
.header(http::header::ACCEPT, APP_JSON)
|
||||
.body(Body::empty())?;
|
||||
let client = hyper::Client::new();
|
||||
let res = client.request(req).await?;
|
||||
if res.status() != StatusCode::OK {
|
||||
error!("client response {:?}", res);
|
||||
}
|
||||
let buf = hyper::body::to_bytes(res.into_body()).await?;
|
||||
let s = String::from_utf8_lossy(&buf);
|
||||
let res: JsonValue = serde_json::from_str(&s)?;
|
||||
info!("GOT: {}", serde_json::to_string_pretty(&res)?);
|
||||
let t2 = chrono::Utc::now();
|
||||
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
|
||||
info!("time {} ms", ms);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user