daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Begin refactor frame handling, update clap

Browse Source
This commit is contained in:
Dominik Werder
2022-11-09 15:50:41 +01:00
parent fc22d1ebaf
commit 9036160253
37 changed files with 466 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
disk/src/binned.rs
View File

@@ -17,16 +17,13 @@ use futures_util::StreamExt;
use items::numops::NumOps;
use items::streams::{Collectable, Collector};
use items::{
Clearable, EventsNodeProcessor, FilterFittingInside, Framable, FrameType, PushableIndex, RangeCompletableItem,
Sitemty, StreamItem, TimeBinnableType, WithLen,
Clearable, EventsNodeProcessor, FilterFittingInside, Framable, FrameDecodable, FrameType, PushableIndex,
RangeCompletableItem, Sitemty, StreamItem, TimeBinnableType, WithLen,
};
use netpod::log::*;
use netpod::query::{BinnedQuery, RawEventsQuery};
use netpod::{
x_bin_count, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, ScalarType,
Shape,
};
use serde::de::DeserializeOwned;
use netpod::x_bin_count;
use netpod::{BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, ScalarType, Shape};
use std::fmt::Debug;
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -72,7 +69,7 @@ impl ChannelExecFunction for BinnedBinaryChannelExec {
+ Unpin,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
FrameType + Framable + DeserializeOwned,
FrameType + Framable + FrameDecodable,
{
let _ = event_value_shape;
let range = BinnedRange::covering_range(self.query.range().clone(), self.query.bin_count())?;
@@ -322,7 +319,7 @@ impl ChannelExecFunction for BinnedJsonChannelExec {
+ Unpin,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
FrameType + Framable + DeserializeOwned,
FrameType + Framable + FrameDecodable,
{
let _ = event_value_shape;
let range = BinnedRange::covering_range(self.query.range().clone(), self.query.bin_count())?;

23
disk/src/binned/binnedfrompbv.rs
View File

@@ -7,14 +7,13 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use http::{StatusCode, Uri};
use items::frame::decode_frame;
use items::{FrameType, FrameTypeStatic, RangeCompletableItem, Sitemty, StreamItem, TimeBinnableType};
use items::{FrameDecodable, FrameType, FrameTypeStaticSYC, TimeBinnableType};
use items::{RangeCompletableItem, Sitemty, StreamItem};
use netpod::log::*;
use netpod::query::CacheUsage;
use netpod::{
x_bin_count, AggKind, AppendToUrl, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts,
PreBinnedPatchIterator, ScalarType, Shape,
};
use serde::de::DeserializeOwned;
use netpod::x_bin_count;
use netpod::PreBinnedPatchIterator;
use netpod::{AggKind, AppendToUrl, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts, ScalarType, Shape};
use std::future::ready;
use std::marker::PhantomData;
use std::pin::Pin;
@@ -34,7 +33,8 @@ pub struct FetchedPreBinned<TBT> {
impl<TBT> FetchedPreBinned<TBT> {
pub fn new(query: &PreBinnedQuery, host: String, port: u16) -> Result<Self, Error>
where
TBT: TimeBinnableType,
TBT: FrameTypeStaticSYC + TimeBinnableType,
Sitemty<TBT>: FrameDecodable,
{
// TODO should not assume http:
let mut url = Url::parse(&format!("http://{host}:{port}/api/4/prebinned"))?;
@@ -53,8 +53,8 @@ impl<TBT> FetchedPreBinned<TBT> {
impl<TBT> Stream for FetchedPreBinned<TBT>
where
TBT: FrameTypeStatic + TimeBinnableType,
Sitemty<TBT>: FrameType + DeserializeOwned,
TBT: FrameTypeStaticSYC + TimeBinnableType,
Sitemty<TBT>: FrameDecodable,
{
type Item = Sitemty<TBT>;
@@ -156,7 +156,7 @@ where
impl<TBT> BinnedFromPreBinned<TBT>
where
TBT: TimeBinnableType<Output = TBT> + Unpin + 'static,
Sitemty<TBT>: FrameType + DeserializeOwned,
Sitemty<TBT>: FrameType + FrameDecodable,
{
pub fn new(
patch_it: PreBinnedPatchIterator,
@@ -252,7 +252,8 @@ where
impl<TBT> Stream for BinnedFromPreBinned<TBT>
where
TBT: TimeBinnableType,
TBT: TimeBinnableType<Output = TBT> + Unpin + 'static,
Sitemty<TBT>: FrameType + FrameDecodable,
{
type Item = Sitemty<TBT>;

15
disk/src/binned/pbv.rs
View File

@@ -11,15 +11,14 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use items::numops::NumOps;
use items::{
Appendable, Clearable, EventsNodeProcessor, EventsTypeAliases, FrameType, PushableIndex, RangeCompletableItem,
ReadableFromFile, Sitemty, StreamItem, TimeBinnableType,
Appendable, Clearable, EventsNodeProcessor, EventsTypeAliases, FrameDecodable, FrameType, PushableIndex,
RangeCompletableItem, ReadableFromFile, Sitemty, StreamItem, TimeBinnableType,
};
use netpod::log::*;
use netpod::query::{CacheUsage, RawEventsQuery};
use netpod::{
x_bin_count, AggKind, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange,
};
use serde::de::DeserializeOwned;
use netpod::x_bin_count;
use netpod::{AggKind, BinnedRange, PreBinnedPatchIterator, PreBinnedPatchRange};
use netpod::{NodeConfigCached, PerfOpts};
use serde::Serialize;
use std::future::Future;
use std::io;
@@ -70,7 +69,7 @@ where
// TODO is this needed:
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType,
// TODO who exactly needs this DeserializeOwned?
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: FrameType + DeserializeOwned,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: FrameType + FrameDecodable,
{
pub fn new(query: PreBinnedQuery, agg_kind: AggKind, node_config: &NodeConfigCached) -> Self {
Self {
@@ -484,7 +483,7 @@ where
<ENP as EventsNodeProcessor>::Output: PushableIndex + Appendable + Clearable,
// TODO needed?
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: FrameType + DeserializeOwned,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: FrameType + FrameDecodable,
{
type Item = Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>;

16
disk/src/binned/prebinned.rs
View File

@@ -12,12 +12,11 @@ use futures_core::Stream;
use futures_util::StreamExt;
use items::numops::{BoolNum, NumOps, StringNum};
use items::{
Appendable, Clearable, EventsNodeProcessor, Framable, FrameType, PushableIndex, RangeCompletableItem, Sitemty,
SitemtyFrameType, StreamItem, TimeBinnableType, TimeBinned,
Appendable, Clearable, EventsNodeProcessor, Framable, FrameDecodable, FrameType, PushableIndex,
RangeCompletableItem, Sitemty, SitemtyFrameType, StreamItem, TimeBinnableType, TimeBinned,
};
use netpod::log::*;
use netpod::{AggKind, ByteOrder, ChannelTyped, NodeConfigCached, ScalarType, Shape};
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::pin::Pin;
@@ -36,10 +35,9 @@ where
EVS: EventValueShape<NTY, END> + EventValueFromBytes<NTY, END> + 'static,
ENP: EventsNodeProcessor<Input = <EVS as EventValueFromBytes<NTY, END>>::Batch> + 'static,
<ENP as EventsNodeProcessor>::Output: PushableIndex + Appendable + Clearable + 'static,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
Framable + FrameType + DeserializeOwned,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output: SitemtyFrameType + TimeBinned,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>: Framable + FrameType + FrameDecodable,
{
if let Some(scyconf) = &node_config.node_config.cluster.cache_scylla {
trace!("~~~~~~~~~~~~~~~ make_num_pipeline_nty_end_evs_enp using scylla as cache");
@@ -58,11 +56,11 @@ where
)
.await?;
let stream = stream.map(|x| {
//
match x {
let ret = match x {
Ok(k) => Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))),
Err(e) => Err(e),
}
};
ret
});
let stream = Box::pin(stream) as Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>;
Ok(stream)

9
disk/src/channelexec.rs
View File

@@ -13,8 +13,8 @@ use items::numops::{BoolNum, NumOps, StringNum};
use items::scalarevents::ScalarEvents;
use items::streams::{Collectable, Collector};
use items::{
Clearable, EventsNodeProcessor, Framable, FrameType, PushableIndex, RangeCompletableItem, Sitemty, StreamItem,
TimeBinnableType,
Clearable, EventsNodeProcessor, Framable, FrameType, FrameTypeStatic, PushableIndex, RangeCompletableItem, Sitemty,
StreamItem, TimeBinnableType,
};
use netpod::log::*;
use netpod::query::{PlainEventsQuery, RawEventsQuery};
@@ -75,9 +75,10 @@ where
+ TimeBinnableType<Output = <<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>
+ Collectable
+ Unpin,
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
// TODO shouldn't one of FrameType or FrameTypeStatic be enough?
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + FrameTypeStatic + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
FrameType + Framable + DeserializeOwned,
FrameType + FrameTypeStatic + Framable + DeserializeOwned,
{
Ok(f.exec(byte_order, scalar_type, shape, event_value_shape, events_node_proc)?)
}

5
disk/src/eventblobs.rs
View File

@@ -76,8 +76,9 @@ impl Stream for EventChunkerMultifile {
type Item = Result<StreamItem<RangeCompletableItem<EventFull>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let span1 = span!(Level::INFO, "EventChunkerMultifile", desc = tracing::field::Empty);
span1.record("desc", &"");
//tracing::field::DebugValue;
let span1 = span!(Level::INFO, "EventChunkerMultifile", node_ix = self.node_ix);
//span1.record("node_ix", &self.node_ix);
span1.in_scope(|| {
use Poll::*;
'outer: loop {

11
disk/src/eventchunker.rs
View File

@@ -4,7 +4,7 @@ use bytes::{Buf, BytesMut};
use err::Error;
use futures_util::{Stream, StreamExt};
use items::{
Appendable, ByteEstimate, Clearable, FrameTypeStatic, PushableIndex, RangeCompletableItem, SitemtyFrameType,
Appendable, ByteEstimate, Clearable, FrameTypeStaticSYC, PushableIndex, RangeCompletableItem, SitemtyFrameType,
StatsItem, StreamItem, WithLen, WithTimestamps,
};
use netpod::histo::HistoLog2;
@@ -528,18 +528,13 @@ impl EventFull {
}
}
impl FrameTypeStatic for EventFull {
impl FrameTypeStaticSYC for EventFull {
const FRAME_TYPE_ID: u32 = items::EVENT_FULL_FRAME_TYPE_ID;
fn from_error(_: err::Error) -> Self {
// TODO remove usage of this
panic!()
}
}
impl SitemtyFrameType for EventFull {
fn frame_type_id(&self) -> u32 {
<Self as FrameTypeStatic>::FRAME_TYPE_ID
<Self as FrameTypeStaticSYC>::FRAME_TYPE_ID
}
}

19
disk/src/raw/client.rs
View File

@@ -11,10 +11,10 @@ use crate::raw::eventsfromframes::EventsFromFrames;
use err::Error;
use futures_core::Stream;
use items::frame::{make_frame, make_term_frame};
use items::{EventsNodeProcessor, FrameType, RangeCompletableItem, Sitemty, StreamItem};
use items::{EventQueryJsonStringFrame, EventsNodeProcessor, RangeCompletableItem, Sitemty, StreamItem};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::{EventQueryJsonStringFrame, Node, PerfOpts};
use netpod::{Node, PerfOpts};
use std::pin::Pin;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
@@ -27,15 +27,17 @@ pub async fn x_processed_stream_from_node<ENP>(
where
ENP: EventsNodeProcessor,
<ENP as EventsNodeProcessor>::Output: Unpin + 'static,
Result<StreamItem<RangeCompletableItem<<ENP as EventsNodeProcessor>::Output>>, err::Error>: FrameType,
{
debug!("x_processed_stream_from_node to: {}:{}", node.host, node.port_raw);
let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
let qjs = serde_json::to_string(&query)?;
let (netin, mut netout) = net.into_split();
let buf = make_frame(&EventQueryJsonStringFrame(qjs))?;
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(
EventQueryJsonStringFrame(qjs),
)));
let buf = make_frame(&item)?;
netout.write_all(&buf).await?;
let buf = make_term_frame();
let buf = make_term_frame()?;
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();
@@ -56,9 +58,12 @@ pub async fn x_processed_event_blobs_stream_from_node(
let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
let qjs = serde_json::to_string(&query)?;
let (netin, mut netout) = net.into_split();
let buf = make_frame(&EventQueryJsonStringFrame(qjs))?;
let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(
EventQueryJsonStringFrame(qjs),
)));
let buf = make_frame(&item)?;
netout.write_all(&buf).await?;
let buf = make_term_frame();
let buf = make_term_frame()?;
netout.write_all(&buf).await?;
netout.flush().await?;
netout.forget();

16
disk/src/raw/conn.rs
View File

@@ -1,12 +1,10 @@
use crate::decode::{
BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValuesDim0Case, EventValuesDim1Case,
EventsDecodedStream, LittleEndian, NumFromBytes,
};
use crate::decode::{BigEndian, Endianness, LittleEndian};
use crate::decode::{EventValueFromBytes, EventValueShape, EventsDecodedStream, NumFromBytes};
use crate::decode::{EventValuesDim0Case, EventValuesDim1Case};
use crate::eventblobs::EventChunkerMultifile;
use crate::eventchunker::{EventChunkerConf, EventFull};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use futures_util::{Stream, StreamExt};
use items::numops::{BoolNum, NumOps, StringNum};
use items::{EventsNodeProcessor, Framable, RangeCompletableItem, Sitemty, StreamItem};
use netpod::log::*;
@@ -319,6 +317,8 @@ pub async fn make_event_blobs_pipe(
let range = &evq.range;
let entry = get_applicable_entry(&evq.range, evq.channel.clone(), node_config).await?;
let event_chunker_conf = EventChunkerConf::new(ByteSize::kb(1024));
type ItemType = Sitemty<EventFull>;
// TODO should depend on host config
let pipe = if true {
let event_blobs = make_remote_event_blobs_stream(
range.clone(),
@@ -330,7 +330,7 @@ pub async fn make_event_blobs_pipe(
evq.disk_io_tune.clone(),
node_config,
)?;
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable + Send>);
let s = event_blobs.map(|item: ItemType| Box::new(item) as Box<dyn Framable + Send>);
//let s = tracing_futures::Instrumented::instrument(s, tracing::info_span!("make_event_blobs_pipe"));
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
pipe = Box::pin(s);
@@ -346,7 +346,7 @@ pub async fn make_event_blobs_pipe(
evq.disk_io_tune.clone(),
node_config,
)?;
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable + Send>);
let s = event_blobs.map(|item: ItemType| Box::new(item) as Box<dyn Framable + Send>);
//let s = tracing_futures::Instrumented::instrument(s, tracing::info_span!("make_event_blobs_pipe"));
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
pipe = Box::pin(s);

4
disk/src/raw/eventsfromframes.rs
View File

@@ -2,7 +2,7 @@ use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use futures_core::Stream;
use futures_util::StreamExt;
use items::frame::decode_frame;
use items::{FrameTypeStatic, Sitemty, StreamItem};
use items::{FrameTypeStaticSYC, Sitemty, StreamItem};
use netpod::log::*;
use serde::de::DeserializeOwned;
use std::marker::PhantomData;
@@ -37,7 +37,7 @@ where
impl<T, I> Stream for EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
I: FrameTypeStatic + DeserializeOwned + Unpin,
I: FrameTypeStaticSYC + DeserializeOwned + Unpin,
{
type Item = Sitemty<I>;