daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove obsolete containers

Browse Source
This commit is contained in:
Dominik Werder
2023-02-08 10:09:17 +01:00
parent 326fe793ce
commit b93bb9b467
24 changed files with 192 additions and 5930 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
daqbufp2/src/client.rs
View File

@@ -1,17 +1,24 @@
use crate::err::ErrConv;
use chrono::{DateTime, Utc};
use chrono::DateTime;
use chrono::Utc;
use disk::streamlog::Streamlog;
use err::Error;
use futures_util::TryStreamExt;
use http::StatusCode;
use httpclient::HttpBodyAsAsyncRead;
use hyper::Body;
use items::xbinnedwaveevents::XBinnedWaveEvents;
use items::{Sitemty, StreamItem};
use items::StreamItem;
use netpod::log::*;
use netpod::query::{BinnedQuery, CacheUsage};
use netpod::query::BinnedQuery;
use netpod::query::CacheUsage;
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::{AggKind, ByteSize, Channel, HostPort, NanoRange, PerfOpts, APP_OCTET};
use netpod::ByteSize;
use netpod::Channel;
use netpod::HostPort;
use netpod::NanoRange;
use netpod::PerfOpts;
use netpod::APP_OCTET;
use streams::frames::inmem::InMemoryFrameAsyncReadStream;
use url::Url;
@@ -110,44 +117,11 @@ pub async fn get_binned(
info!("Stats: {:?}", item);
None
}
StreamItem::DataItem(frame) => {
StreamItem::DataItem(_frame) => {
// TODO
// The expected type nowadays depends on the channel and agg-kind.
err::todo();
type ExpectedType = Sitemty<XBinnedWaveEvents<u8>>;
// TODO the non-data variants of Sitemty no longer carry a frame id.
//let type_id_exp = <ExpectedType as FrameType>::FRAME_TYPE_ID;
let type_id_exp: u32 = err::todoval();
if frame.tyid() != type_id_exp {
error!("unexpected type id got {} exp {}", frame.tyid(), type_id_exp);
}
let n1 = frame.buf().len();
match rmp_serde::from_slice::<ExpectedType>(frame.buf()) {
Ok(item) => match item {
Ok(item) => {
match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
}
StreamItem::Stats(item) => {
info!("Stats: {:?}", item);
}
StreamItem::DataItem(item) => {
info!("DataItem: {:?}", item);
}
}
Some(Ok(()))
}
Err(e) => {
error!("len {} error frame {:?}", n1, e);
Some(Err(e))
}
},
Err(e) => {
error!("len {} {:?}", n1, e);
Some(Err(e.into()))
}
}
Some(Ok(()))
}
},
Err(e) => Some(Err(Error::with_msg(format!("{:?}", e)))),

82
daqbufp2/src/test/binnedbinary.rs
View File

@@ -1,19 +1,27 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use chrono::{DateTime, Utc};
use chrono::DateTime;
use chrono::Utc;
use disk::streamlog::Streamlog;
use err::Error;
use futures_util::{StreamExt, TryStreamExt};
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use http::StatusCode;
use httpclient::HttpBodyAsAsyncRead;
use hyper::Body;
use items::binsdim0::MinMaxAvgDim0Bins;
use items::{RangeCompletableItem, Sitemty, StatsItem, StreamItem, WithLen};
use items::StreamItem;
use items_0::subfr::SubFrId;
use netpod::log::*;
use netpod::query::{BinnedQuery, CacheUsage};
use netpod::query::BinnedQuery;
use netpod::query::CacheUsage;
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::{AggKind, Channel, Cluster, HostPort, NanoRange, PerfOpts, APP_OCTET};
use netpod::Channel;
use netpod::Cluster;
use netpod::HostPort;
use netpod::NanoRange;
use netpod::PerfOpts;
use netpod::APP_OCTET;
use serde::de::DeserializeOwned;
use std::fmt;
use std::future::ready;
@@ -144,6 +152,7 @@ where
}
}
#[allow(unused)]
#[derive(Debug)]
pub struct BinnedResponse {
bin_count: u64,
@@ -178,6 +187,7 @@ impl BinnedResponse {
}
}
// TODO
async fn consume_binned_response<NTY, T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<BinnedResponse, Error>
where
NTY: fmt::Debug + SubFrId + DeserializeOwned,
@@ -197,68 +207,16 @@ where
debug!("Stats: {:?}", item);
None
}
StreamItem::DataItem(frame) => {
// TODO non-data Sitety no longer carry frame id:
//if frame.tyid() != <Sitemty<MinMaxAvgDim0Bins<NTY>> as FrameType>::FRAME_TYPE_ID {
if frame.tyid() != err::todoval::<u32>() {
error!("test receives unexpected tyid {:x}", frame.tyid());
}
match rmp_serde::from_slice::<Sitemty<MinMaxAvgDim0Bins<NTY>>>(frame.buf()) {
Ok(item) => match item {
Ok(item) => match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
Some(Ok(StreamItem::Log(item)))
}
item => Some(Ok(item)),
},
Err(e) => {
error!("TEST GOT ERROR FRAME: {:?}", e);
Some(Err(e))
}
},
Err(e) => {
error!("{:?}", e);
Some(Err(e.into()))
}
}
StreamItem::DataItem(_frame) => {
err::todo();
Some(Ok(()))
}
},
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
};
ready(g)
})
.fold(BinnedResponse::new(), |mut a, k| {
let g = match k {
Ok(StreamItem::Log(_item)) => {
a.log_item_count += 1;
a
}
Ok(StreamItem::Stats(item)) => match item {
StatsItem::EventDataReadStats(item) => {
a.bytes_read += item.parsed_bytes;
a
}
_ => a,
},
Ok(StreamItem::DataItem(item)) => match item {
RangeCompletableItem::RangeComplete => {
a.range_complete_count += 1;
a
}
RangeCompletableItem::Data(item) => {
a.data_item_count += 1;
a.bin_count += WithLen::len(&item) as u64;
a
}
},
Err(_e) => {
a.err_item_count += 1;
a
}
};
ready(g)
});
.fold(BinnedResponse::new(), |a, _x| ready(a));
let ret = s1.await;
debug!("BinnedResponse: {:?}", ret);
Ok(ret)

99
daqbufp2/src/test/events.rs
View File

@@ -1,18 +1,26 @@
use crate::err::ErrConv;
use crate::nodes::require_test_hosts_running;
use chrono::{DateTime, Utc};
use chrono::DateTime;
use chrono::Utc;
use disk::streamlog::Streamlog;
use err::Error;
use futures_util::{StreamExt, TryStreamExt};
use futures_util::StreamExt;
use futures_util::TryStreamExt;
use http::StatusCode;
use httpclient::HttpBodyAsAsyncRead;
use hyper::Body;
use items::numops::NumOps;
use items::scalarevents::ScalarEvents;
use items::{RangeCompletableItem, Sitemty, StatsItem, StreamItem, WithLen};
use items::StreamItem;
use netpod::log::*;
use netpod::query::PlainEventsQuery;
use netpod::{log::*, AggKind};
use netpod::{AppendToUrl, Channel, Cluster, HostPort, NanoRange, PerfOpts, APP_JSON, APP_OCTET};
use netpod::AggKind;
use netpod::AppendToUrl;
use netpod::Channel;
use netpod::Cluster;
use netpod::HostPort;
use netpod::NanoRange;
use netpod::PerfOpts;
use netpod::APP_JSON;
use netpod::APP_OCTET;
use serde_json::Value as JsonValue;
use std::fmt::Debug;
use std::future::ready;
@@ -42,7 +50,7 @@ async fn get_plain_events_binary_0_inner() -> Result<(), Error> {
let rh = require_test_hosts_running()?;
let cluster = &rh.cluster;
if true {
get_plain_events_binary::<i32>(
get_plain_events_binary(
"scalar-i32-be",
"1970-01-01T00:20:10.000Z",
"1970-01-01T00:20:50.000Z",
@@ -60,17 +68,14 @@ fn get_plain_events_binary_0() {
taskrun::run(get_plain_events_binary_0_inner()).unwrap();
}
async fn get_plain_events_binary<NTY>(
async fn get_plain_events_binary(
channel_name: &str,
beg_date: &str,
end_date: &str,
cluster: &Cluster,
_expect_range_complete: bool,
_expect_event_count: u64,
) -> Result<EventsResponse, Error>
where
NTY: NumOps,
{
) -> Result<EventsResponse, Error> {
let t1 = Utc::now();
let node0 = &cluster.nodes[0];
let beg_date: DateTime<Utc> = beg_date.parse()?;
@@ -110,7 +115,7 @@ where
}
let s1 = HttpBodyAsAsyncRead::new(res);
let s2 = InMemoryFrameAsyncReadStream::new(s1, perf_opts.inmem_bufcap);
let res = consume_plain_events_binary::<NTY, _>(s2).await?;
let res = consume_plain_events_binary(s2).await?;
let t2 = chrono::Utc::now();
let ms = t2.signed_duration_since(t1).num_milliseconds() as u64;
// TODO add timeout
@@ -122,6 +127,7 @@ where
}
}
#[allow(unused)]
#[derive(Debug)]
pub struct EventsResponse {
event_count: u64,
@@ -156,9 +162,8 @@ impl EventsResponse {
}
}
async fn consume_plain_events_binary<NTY, T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<EventsResponse, Error>
async fn consume_plain_events_binary<T>(inp: InMemoryFrameAsyncReadStream<T>) -> Result<EventsResponse, Error>
where
NTY: NumOps,
T: AsyncRead + Unpin,
{
let s1 = inp
@@ -174,70 +179,16 @@ where
debug!("Stats: {:?}", item);
None
}
StreamItem::DataItem(frame) => {
// TODO the non-data variants of Sitemty no longer carry frame type id:
//if frame.tyid() != <Sitemty<ScalarEvents<NTY>> as FrameType>::FRAME_TYPE_ID {
if frame.tyid() != err::todoval::<u32>() {
error!("test receives unexpected tyid {:x}", frame.tyid());
None
} else {
match rmp_serde::from_slice::<Sitemty<ScalarEvents<NTY>>>(frame.buf()) {
Ok(item) => match item {
Ok(item) => match item {
StreamItem::Log(item) => {
Streamlog::emit(&item);
Some(Ok(StreamItem::Log(item)))
}
item => Some(Ok(item)),
},
Err(e) => {
error!("TEST GOT ERROR FRAME: {:?}", e);
Some(Err(e))
}
},
Err(e) => {
error!("{:?}", e);
Some(Err(e.into()))
}
}
}
StreamItem::DataItem(_frame) => {
err::todo();
Some(Ok(()))
}
},
Err(e) => Some(Err(Error::with_msg(format!("WEIRD EMPTY ERROR {:?}", e)))),
};
ready(g)
})
.fold(EventsResponse::new(), |mut a, k| {
let g = match k {
Ok(StreamItem::Log(_item)) => {
a.log_item_count += 1;
a
}
Ok(StreamItem::Stats(item)) => match item {
StatsItem::EventDataReadStats(item) => {
a.bytes_read += item.parsed_bytes;
a
}
_ => a,
},
Ok(StreamItem::DataItem(item)) => match item {
RangeCompletableItem::RangeComplete => {
a.range_complete_count += 1;
a
}
RangeCompletableItem::Data(item) => {
a.data_item_count += 1;
a.event_count += WithLen::len(&item) as u64;
a
}
},
Err(_e) => {
a.err_item_count += 1;
a
}
};
ready(g)
});
.fold(EventsResponse::new(), |a, _x| ready(a));
let ret = s1.await;
debug!("result: {:?}", ret);
Ok(ret)