daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add BoolNum type and test on slow GLS

Browse Source
This commit is contained in:
Dominik Werder
2021-06-24 08:46:56 +02:00
parent 411014d289
commit ef803a45a2
19 changed files with 249 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
disk/src/aggtest.rs
View File

@@ -15,6 +15,7 @@ pub fn make_test_node(id: u32) -> Node {
split: id,
ksprefix: "ks".into(),
backend: "testbackend".into(),
bin_grain_kind: 0,
}
}

86
disk/src/binned.rs
View File

@@ -9,8 +9,8 @@ use crate::binned::query::BinnedQuery;
use crate::binnedstream::BoxedStream;
use crate::channelexec::{channel_exec, collect_plain_events_json, ChannelExecFunction};
use crate::decode::{
BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValues, EventValuesDim0Case, EventValuesDim1Case,
LittleEndian, NumFromBytes,
Endianness, EventValueFromBytes, EventValueShape, EventValues, EventValuesDim0Case, EventValuesDim1Case,
NumFromBytes,
};
use crate::frame::makeframe::{Framable, FrameType, SubFrId};
use crate::merge::mergedfromremotes::MergedFromRemotes;
@@ -24,8 +24,8 @@ use futures_util::{FutureExt, StreamExt};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{
x_bin_count, AggKind, BinnedRange, ByteOrder, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator,
PreBinnedPatchRange, ScalarType, Shape,
x_bin_count, AggKind, BinnedRange, BoolNum, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator,
PreBinnedPatchRange, Shape,
};
use num_traits::{AsPrimitive, Bounded, Float, Zero};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
@@ -84,11 +84,20 @@ where
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output: Sized,
{
let _ = event_value_shape;
let range = BinnedRange::covering_range(query.range().clone(), query.bin_count())?.ok_or(Error::with_msg(
format!("binned_bytes_for_http BinnedRange::covering_range returned None"),
))?;
let range = BinnedRange::covering_range(
query.range().clone(),
query.bin_count(),
node_config.node.bin_grain_kind,
)?
.ok_or(Error::with_msg(format!(
"binned_bytes_for_http BinnedRange::covering_range returned None"
)))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
match PreBinnedPatchRange::covering_range(query.range().clone(), query.bin_count()) {
match PreBinnedPatchRange::covering_range(
query.range().clone(),
query.bin_count(),
node_config.node.bin_grain_kind,
) {
Ok(Some(pre_range)) => {
info!("binned_bytes_for_http found pre_range: {:?}", pre_range);
if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() {
@@ -151,6 +160,7 @@ pub struct BinnedResponseDyn {
stream: Pin<Box<dyn Stream<Item = Box<dyn BinnedResponseItem>> + Send>>,
}
// TODO remove after refactor of PPP:
fn make_num_pipeline_nty_end_evs_enp<PPP, NTY, END, EVS, ENP>(
shape: Shape,
_agg_kind: AggKind,
@@ -187,6 +197,8 @@ where
Ok(ret)
}
// TODO remove after refactor of PPP:
#[allow(dead_code)]
fn make_num_pipeline_nty_end<PPP, NTY, END>(
shape: Shape,
query: BinnedQuery,
@@ -249,8 +261,8 @@ where
)
}
AggKind::DimXBinsN(_) => {
/*let events_node_proc = <<EventValuesDim1Case<NTY> as EventValueShape<NTY, END>>::NumXAggToNBins as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
let yo = make_num_pipeline_nty_end_evs_enp::<PPP, NTY, END, _, _>(
let _events_node_proc = <<EventValuesDim1Case<NTY> as EventValueShape<NTY, END>>::NumXAggToNBins as EventsNodeProcessor>::create(shape.clone(), agg_kind.clone());
/*let yo = make_num_pipeline_nty_end_evs_enp::<PPP, NTY, END, _, _>(
shape,
agg_kind,
evs,
@@ -269,6 +281,7 @@ where
}
}
// TODO remove after refactor of PPP:
#[allow(dead_code)]
fn make_num_pipeline_nty_end_old<PPP, NTY, END>(
shape: Shape,
@@ -306,6 +319,8 @@ where
}
}
// TODO remove after refactor of PPP:
#[allow(unused_macros)]
macro_rules! match_end {
($nty:ident, $end:expr, $shape:expr, $query:expr, $ppp:expr, $node_config:expr) => {
match $end {
@@ -315,7 +330,8 @@ macro_rules! match_end {
};
}
fn make_num_pipeline_entry<PPP>(
// TODO remove after refactor of PPP
/*fn make_num_pipeline_entry<PPP>(
scalar_type: ScalarType,
byte_order: ByteOrder,
shape: Shape,
@@ -347,27 +363,15 @@ where
ScalarType::I64 => match_end!(i64, byte_order, shape, query, ppp, node_config),
ScalarType::F32 => match_end!(f32, byte_order, shape, query, ppp, node_config),
ScalarType::F64 => match_end!(f64, byte_order, shape, query, ppp, node_config),
ScalarType::BOOL => match_end!(BoolNum, byte_order, shape, query, ppp, node_config),
}
}
}*/
async fn make_num_pipeline<PPP>(
query: &BinnedQuery,
ppp: PPP,
_ppp: PPP,
node_config: &NodeConfigCached,
) -> Result<BinnedResponseDyn, Error>
where
PPP: PipelinePostProcessA,
PPP: PipelinePostProcessB<MinMaxAvgBins<u8>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<u16>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<u32>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<u64>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<i8>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<i16>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<i32>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<i64>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<f32>>,
PPP: PipelinePostProcessB<MinMaxAvgBins<f64>>,
{
) -> Result<BinnedResponseDyn, Error> {
if query.channel().backend != node_config.node.backend {
let err = Error::with_msg(format!(
"backend mismatch node: {} requested: {}",
@@ -400,7 +404,7 @@ where
MatchingConfigEntry::Entry(entry) => {
// TODO make this a stream log:
info!("binned_bytes_for_http found config entry {:?}", entry);
let ret = make_num_pipeline_entry(
/*let ret = make_num_pipeline_entry(
entry.scalar_type.clone(),
entry.byte_order.clone(),
entry.to_shape()?,
@@ -408,7 +412,8 @@ where
ppp,
node_config,
)?;
Ok(ret)
Ok(ret)*/
err::todoval()
}
}
}
@@ -747,13 +752,21 @@ impl ChannelExecFunction for BinnedJsonChannelExec {
FrameType + Framable + DeserializeOwned,
{
let _ = event_value_shape;
let range =
BinnedRange::covering_range(self.query.range().clone(), self.query.bin_count())?.ok_or(Error::with_msg(
format!("binned_bytes_for_http BinnedRange::covering_range returned None"),
))?;
let range = BinnedRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
)?
.ok_or(Error::with_msg(format!(
"binned_bytes_for_http BinnedRange::covering_range returned None"
)))?;
let t_bin_count = range.count as u32;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let souter = match PreBinnedPatchRange::covering_range(self.query.range().clone(), self.query.bin_count()) {
let souter = match PreBinnedPatchRange::covering_range(
self.query.range().clone(),
self.query.bin_count(),
self.node_config.node.bin_grain_kind,
) {
Ok(Some(pre_range)) => {
info!("binned_bytes_for_http found pre_range: {:?}", pre_range);
if range.grid_spec.bin_t_len() < pre_range.grid_spec.bin_t_len() {
@@ -982,7 +995,6 @@ pub trait NumOps:
fn min_or_nan() -> Self;
fn max_or_nan() -> Self;
fn is_nan(&self) -> bool;
fn fourty_two() -> Self;
}
macro_rules! impl_num_ops {
@@ -997,9 +1009,6 @@ macro_rules! impl_num_ops {
fn is_nan(&self) -> bool {
$is_nan(self)
}
fn fourty_two() -> Self {
42 as Self
}
}
};
}
@@ -1022,6 +1031,7 @@ impl_num_ops!(i32, MIN, MAX, is_nan_int);
impl_num_ops!(i64, MIN, MAX, is_nan_int);
impl_num_ops!(f32, NAN, NAN, is_nan_float);
impl_num_ops!(f64, NAN, NAN, is_nan_float);
impl_num_ops!(BoolNum, MIN, MAX, is_nan_int);
pub trait EventsDecoder {
type Output;

18
disk/src/binned/pbv.rs
View File

@@ -126,7 +126,7 @@ where
}
// TODO do I need to set up more transformations or binning to deliver the requested data?
let count = self.query.patch().patch_t_len() / self.query.patch().bin_t_len();
let range = BinnedRange::covering_range(evq.range.clone(), count as u32)?
let range = BinnedRange::covering_range(evq.range.clone(), count as u32, self.node_config.node.bin_grain_kind)?
.ok_or(Error::with_msg("covering_range returns None"))?;
let perf_opts = PerfOpts { inmem_bufcap: 512 };
let s = MergedFromRemotes::<ENP>::new(evq, perf_opts, self.node_config.node_config.cluster.clone());
@@ -204,7 +204,11 @@ where
fn try_setup_fetch_prebinned_higher_res(&mut self) -> Result<(), Error> {
let range = self.query.patch().patch_range();
match PreBinnedPatchRange::covering_range(range, self.query.patch().bin_count() + 1) {
match PreBinnedPatchRange::covering_range(
range,
self.query.patch().bin_count() + 1,
self.node_config.node.bin_grain_kind,
) {
Ok(Some(range)) => {
self.fut2 = Some(self.setup_from_higher_res_prebinned(range)?);
}
@@ -247,8 +251,14 @@ where
self.write_fut = None;
match item {
Ok(res) => {
self.streamlog
.append(Level::INFO, format!("cache file written bytes: {}", res.bytes));
self.streamlog.append(
Level::INFO,
format!(
"cache file written bytes: {} duration {} ms",
res.bytes,
res.duration.as_millis()
),
);
continue 'outer;
}
Err(e) => {

3
disk/src/binned/prebinned.rs
View File

@@ -14,7 +14,7 @@ use bytes::Bytes;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::{AggKind, ByteOrder, NodeConfigCached, ScalarType, Shape};
use netpod::{AggKind, BoolNum, ByteOrder, NodeConfigCached, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use serde::de::DeserializeOwned;
use serde::Serialize;
@@ -145,6 +145,7 @@ fn make_num_pipeline(
ScalarType::I64 => match_end!(i64, byte_order, shape, agg_kind, query, node_config),
ScalarType::F32 => match_end!(f32, byte_order, shape, agg_kind, query, node_config),
ScalarType::F64 => match_end!(f64, byte_order, shape, agg_kind, query, node_config),
ScalarType::BOOL => match_end!(BoolNum, byte_order, shape, agg_kind, query, node_config),
}
}

9
disk/src/cache.rs
View File

@@ -12,6 +12,7 @@ use std::io;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tiny_keccak::Hasher;
use tokio::io::{AsyncRead, ReadBuf};
@@ -154,6 +155,7 @@ impl CacheFileDesc {
pub struct WrittenPbCache {
pub bytes: u64,
pub duration: Duration,
}
pub async fn write_pb_cache_min_max_avg_scalar<T>(
@@ -173,6 +175,7 @@ where
};
let path = cfd.path(&node_config);
let enc = serde_cbor::to_vec(&values)?;
let ts1 = Instant::now();
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
let now = Utc::now();
let mut h = crc32fast::Hasher::new();
@@ -205,7 +208,11 @@ where
})
.await??;
tokio::fs::rename(&tmp_path, &path).await?;
let ret = WrittenPbCache { bytes: res as u64 };
let ts2 = Instant::now();
let ret = WrittenPbCache {
bytes: res as u64,
duration: ts2.duration_since(ts1),
};
Ok(ret)
}

3
disk/src/channelexec.rs
View File

@@ -16,7 +16,7 @@ use futures_core::Stream;
use futures_util::future::FutureExt;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{AggKind, ByteOrder, Channel, NanoRange, NodeConfigCached, PerfOpts, ScalarType, Shape};
use netpod::{AggKind, BoolNum, ByteOrder, Channel, NanoRange, NodeConfigCached, PerfOpts, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use serde::de::DeserializeOwned;
use serde_json::Value as JsonValue;
@@ -161,6 +161,7 @@ where
ScalarType::I64 => match_end!(f, i64, byte_order, shape, agg_kind, node_config),
ScalarType::F32 => match_end!(f, f32, byte_order, shape, agg_kind, node_config),
ScalarType::F64 => match_end!(f, f64, byte_order, shape, agg_kind, node_config),
ScalarType::BOOL => match_end!(f, BoolNum, byte_order, shape, agg_kind, node_config),
}
}

8
disk/src/dataopen.rs
View File

@@ -5,6 +5,7 @@ use futures_util::StreamExt;
use netpod::log::*;
use netpod::{ChannelConfig, NanoRange, Nanos, Node};
use std::path::PathBuf;
use std::time::Instant;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncReadExt, AsyncSeekExt, ErrorKind, SeekFrom};
@@ -125,7 +126,14 @@ async fn open_files_inner(
}
Err(e) => match e.kind() {
ErrorKind::NotFound => {
let ts1 = Instant::now();
let res = super::index::position_static_len_datafile(file, range.beg).await?;
let ts2 = Instant::now();
if false {
// TODO collect for stats:
let dur = ts2.duration_since(ts1);
info!("position_static_len_datafile took ms {}", dur.as_millis());
}
file = res.0;
if res.1 {
OpenedFile {

14
disk/src/decode.rs
View File

@@ -11,7 +11,7 @@ use crate::eventchunker::EventFull;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::NanoRange;
use netpod::{BoolNum, NanoRange};
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;
use std::mem::size_of;
@@ -39,6 +39,18 @@ pub trait NumFromBytes<NTY, END> {
fn convert(buf: &[u8], big_endian: bool) -> NTY;
}
impl NumFromBytes<BoolNum, LittleEndian> for BoolNum {
fn convert(buf: &[u8], _big_endian: bool) -> BoolNum {
BoolNum(buf[0])
}
}
impl NumFromBytes<BoolNum, BigEndian> for BoolNum {
fn convert(buf: &[u8], _big_endian: bool) -> BoolNum {
BoolNum(buf[0])
}
}
macro_rules! impl_num_from_bytes_end {
($nty:ident, $nl:expr, $end:ident, $ec:ident) => {
impl NumFromBytes<$nty, $end> for $nty {

4
disk/src/eventchunker.rs
View File

@@ -385,6 +385,10 @@ impl Stream for EventChunker {
} else {
match self.inp.poll_next_unpin(cx) {
Ready(Some(Ok(mut fcr))) => {
if false {
// TODO collect for stats:
info!("file read bytes {} ms {}", fcr.buf.len(), fcr.duration.as_millis());
}
let r = self.parse_buf(&mut fcr.buf);
match r {
Ok(res) => {

5
disk/src/frame/makeframe.rs
View File

@@ -10,6 +10,7 @@ use crate::raw::EventQueryJsonStringFrame;
use crate::Sitemty;
use bytes::{BufMut, BytesMut};
use err::Error;
use netpod::BoolNum;
use serde::{de::DeserializeOwned, Serialize};
pub const INMEM_FRAME_ENCID: u32 = 0x12121212;
@@ -61,6 +62,10 @@ impl SubFrId for f64 {
const SUB: u32 = 12;
}
impl SubFrId for BoolNum {
const SUB: u32 = 13;
}
pub trait FrameType {
const FRAME_TYPE_ID: u32;
}

1
disk/src/gen.rs
View File

@@ -80,6 +80,7 @@ pub async fn gen_test_data() -> Result<(), Error> {
data_base_path: data_base_path.join(format!("node{:02}", i1)),
ksprefix: ksprefix.clone(),
backend: "testbackend".into(),
bin_grain_kind: 0,
};
ensemble.nodes.push(node);
}

27
disk/src/index.rs
View File

@@ -1,31 +1,10 @@
use arrayref::array_ref;
use err::Error;
use netpod::log::*;
use netpod::{ChannelConfig, Nanos, Node};
use netpod::Nanos;
use std::mem::size_of;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncReadExt, AsyncSeekExt, ErrorKind, SeekFrom};
pub async fn find_start_pos_for_config(
ts: Nanos,
channel_config: &ChannelConfig,
node: &Node,
) -> Result<Option<u64>, Error> {
let index_path = super::paths::index_path(ts, channel_config, node)?;
let ret = match OpenOptions::new().open(&index_path).await {
Ok(_file) => {
info!("opened index file");
error!("??????????????? TODO search index for start");
err::todoval::<u32>();
None
}
Err(e) => match e.kind() {
ErrorKind::NotFound => None,
_ => Err(e)?,
},
};
Ok(ret)
}
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom};
pub fn find_ge(h: u64, buf: &[u8]) -> Result<Option<(u64, u64)>, Error> {
const N: usize = 2 * size_of::<u64>();

6
disk/src/lib.rs
View File

@@ -316,10 +316,10 @@ pub fn file_content_stream(
async_stream::stream! {
use tokio::io::AsyncReadExt;
loop {
let ts1 = Instant::now();
let mut buf = BytesMut::with_capacity(buffer_size);
let inst1 = Instant::now();
let n1 = file.read_buf(&mut buf).await?;
let inst2 = Instant::now();
let ts2 = Instant::now();
if n1 == 0 {
info!("file EOF");
break;
@@ -327,7 +327,7 @@ pub fn file_content_stream(
else {
let ret = FileChunkRead {
buf,
duration: inst2.duration_since(inst1),
duration: ts2.duration_since(ts1),
};
yield Ok(ret);
}

7
disk/src/raw/conn.rs
View File

@@ -15,7 +15,7 @@ use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{AggKind, ByteOrder, ByteSize, NodeConfigCached, PerfOpts, ScalarType, Shape};
use netpod::{AggKind, BoolNum, ByteOrder, ByteSize, NodeConfigCached, PerfOpts, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
use std::io;
use std::net::SocketAddr;
@@ -201,6 +201,7 @@ macro_rules! pipe1 {
ScalarType::I64 => pipe2!(i64, $end, $shape, $agg_kind, $event_blobs),
ScalarType::F32 => pipe2!(f32, $end, $shape, $agg_kind, $event_blobs),
ScalarType::F64 => pipe2!(f64, $end, $shape, $agg_kind, $event_blobs),
ScalarType::BOOL => pipe2!(BoolNum, $end, $shape, $agg_kind, $event_blobs),
}
};
}
@@ -298,10 +299,6 @@ async fn events_conn_handler_inner_try(
event_chunker_conf,
);
let shape = entry.to_shape().unwrap();
info!(
"+++++--- conn.rs call pipe1 shape {:?} agg_kind {:?}",
shape, evq.agg_kind
);
let mut p1 = pipe1!(entry.scalar_type, entry.byte_order, shape, evq.agg_kind, event_blobs);
while let Some(item) = p1.next().await {
//info!("conn.rs encode frame typeid {:x}", item.typeid());