daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP refactor frame type id, it type checks

Browse Source
This commit is contained in:
Dominik Werder
2022-06-23 13:33:07 +02:00
parent c046303c7f
commit 66215f583f
29 changed files with 453 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
disk/src/binned/binnedfrompbv.rs
View File

@@ -7,7 +7,7 @@ use futures_core::Stream;
use futures_util::{FutureExt, StreamExt};
use http::{StatusCode, Uri};
use items::frame::decode_frame;
use items::{FrameType, RangeCompletableItem, Sitemty, StreamItem, TimeBinnableType};
use items::{FrameType, FrameTypeStatic, RangeCompletableItem, Sitemty, StreamItem, TimeBinnableType};
use netpod::log::*;
use netpod::query::CacheUsage;
use netpod::{
@@ -32,7 +32,10 @@ pub struct FetchedPreBinned<TBT> {
}
impl<TBT> FetchedPreBinned<TBT> {
pub fn new(query: &PreBinnedQuery, host: String, port: u16) -> Result<Self, Error> {
pub fn new(query: &PreBinnedQuery, host: String, port: u16) -> Result<Self, Error>
where
TBT: TimeBinnableType,
{
// TODO should not assume http:
let mut url = Url::parse(&format!("http://{host}:{port}/api/4/prebinned"))?;
query.append_to_url(&mut url);
@@ -50,7 +53,7 @@ impl<TBT> FetchedPreBinned<TBT> {
impl<TBT> Stream for FetchedPreBinned<TBT>
where
TBT: TimeBinnableType,
TBT: FrameTypeStatic + TimeBinnableType,
Sitemty<TBT>: FrameType + DeserializeOwned,
{
type Item = Sitemty<TBT>;

38
disk/src/binned/prebinned.rs
View File

@@ -12,7 +12,8 @@ use futures_core::Stream;
use futures_util::StreamExt;
use items::numops::{BoolNum, NumOps, StringNum};
use items::{
Appendable, Clearable, EventsNodeProcessor, Framable, FrameType, PushableIndex, Sitemty, TimeBinnableType,
Appendable, Clearable, EventsNodeProcessor, Framable, FrameType, PushableIndex, RangeCompletableItem, Sitemty,
SitemtyFrameType, StreamItem, TimeBinnableType, TimeBinned,
};
use netpod::log::*;
use netpod::{AggKind, ByteOrder, ChannelTyped, NodeConfigCached, ScalarType, Shape};
@@ -28,7 +29,7 @@ async fn make_num_pipeline_nty_end_evs_enp<NTY, END, EVS, ENP>(
_events_node_proc: ENP,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error>
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
END: Endianness + 'static,
@@ -38,6 +39,7 @@ where
Sitemty<<ENP as EventsNodeProcessor>::Output>: FrameType + Framable + 'static,
Sitemty<<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output>:
Framable + FrameType + DeserializeOwned,
<<ENP as EventsNodeProcessor>::Output as TimeBinnableType>::Output: SitemtyFrameType + TimeBinned,
{
if let Some(scyconf) = &node_config.node_config.cluster.cache_scylla {
info!("~~~~~~~~~~~~~~~ make_num_pipeline_nty_end_evs_enp using scylla as cache");
@@ -50,21 +52,29 @@ where
let stream = stream.map(|x| {
//
match x {
Ok(k) => {
let g = Box::new(k) as Box<dyn Framable>;
g
}
Err(e) => {
let u: Sitemty<items::scalarevents::ScalarEvents<f32>> = Err(e);
Box::new(u) as Box<dyn Framable>
}
Ok(k) => Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))),
Err(e) => Err(e),
}
});
let stream = Box::pin(stream) as Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
let stream = Box::pin(stream) as Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>;
Ok(stream)
} else {
let ret = PreBinnedValueStream::<NTY, END, EVS, ENP>::new(query, agg_kind, node_config);
let ret = StreamExt::map(ret, |item| Box::new(item) as Box<dyn Framable>);
let ret = StreamExt::map(ret, |item| {
//
match item {
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))) => {
let g = Box::new(k) as Box<dyn TimeBinned>;
Ok(StreamItem::DataItem(RangeCompletableItem::Data(g)))
}
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete)) => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
Ok(StreamItem::Log(k)) => Ok(StreamItem::Log(k)),
Ok(StreamItem::Stats(k)) => Ok(StreamItem::Stats(k)),
Err(e) => Err(e),
}
});
Ok(Box::pin(ret))
}
}
@@ -75,7 +85,7 @@ async fn make_num_pipeline_nty_end<NTY, END>(
agg_kind: AggKind,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error>
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>, Error>
where
NTY: NumOps + NumFromBytes<NTY, END> + Serialize + 'static,
END: Endianness + 'static,
@@ -188,7 +198,7 @@ async fn make_num_pipeline(
agg_kind: AggKind,
query: PreBinnedQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error> {
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<Box<dyn TimeBinned>>> + Send>>, Error> {
match scalar_type {
ScalarType::U8 => match_end!(u8, byte_order, scalar_type, shape, agg_kind, query, node_config),
ScalarType::U16 => match_end!(u16, byte_order, scalar_type, shape, agg_kind, query, node_config),

4
disk/src/channelexec.rs
View File

@@ -243,7 +243,7 @@ impl PlainEvents {
}
impl ChannelExecFunction for PlainEvents {
type Output = Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
type Output = Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
fn exec<NTY, END, EVS, ENP>(
self,
@@ -265,7 +265,7 @@ impl ChannelExecFunction for PlainEvents {
// TODO let upstream provide DiskIoTune and pass in RawEventsQuery:
let evq = RawEventsQuery::new(self.channel, self.range, self.agg_kind);
let s = MergedFromRemotes::<Identity<NTY>>::new(evq, perf_opts, self.node_config.node_config.cluster);
let s = s.map(|item| Box::new(item) as Box<dyn Framable>);
let s = s.map(|item| Box::new(item) as Box<dyn Framable + Send>);
Ok(Box::pin(s))
}

17
disk/src/eventchunker.rs
View File

@@ -4,8 +4,8 @@ use bytes::{Buf, BytesMut};
use err::Error;
use futures_util::{Stream, StreamExt};
use items::{
Appendable, ByteEstimate, Clearable, PushableIndex, RangeCompletableItem, SitemtyFrameType, StatsItem, StreamItem,
WithLen, WithTimestamps,
Appendable, ByteEstimate, Clearable, FrameTypeStatic, PushableIndex, RangeCompletableItem, SitemtyFrameType,
StatsItem, StreamItem, WithLen, WithTimestamps,
};
use netpod::histo::HistoLog2;
use netpod::log::*;
@@ -528,8 +528,19 @@ impl EventFull {
}
}
impl SitemtyFrameType for EventFull {
impl FrameTypeStatic for EventFull {
const FRAME_TYPE_ID: u32 = items::EVENT_FULL_FRAME_TYPE_ID;
fn from_error(_: err::Error) -> Self {
// TODO remove usage of this
panic!()
}
}
impl SitemtyFrameType for EventFull {
fn frame_type_id(&self) -> u32 {
<Self as FrameTypeStatic>::FRAME_TYPE_ID
}
}
impl WithLen for EventFull {

17
disk/src/raw/conn.rs
View File

@@ -12,7 +12,6 @@ use items::{EventsNodeProcessor, Framable, RangeCompletableItem, Sitemty, Stream
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::{AggKind, ByteOrder, ByteSize, Channel, DiskIoTune, NanoRange, NodeConfigCached, ScalarType, Shape};
use parse::channelconfig::{extract_matching_config_entry, read_local_config, ConfigEntry, MatchingConfigEntry};
use std::pin::Pin;
@@ -20,7 +19,7 @@ fn make_num_pipeline_stream_evs<NTY, END, EVS, ENP>(
event_value_shape: EVS,
events_node_proc: ENP,
event_blobs: EventChunkerMultifile,
) -> Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>
) -> Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>
where
NTY: NumOps + NumFromBytes<NTY, END> + 'static,
END: Endianness + 'static,
@@ -44,7 +43,7 @@ where
},
Err(e) => Err(e),
})
.map(|item| Box::new(item) as Box<dyn Framable>);
.map(|item| Box::new(item) as Box<dyn Framable + Send>);
Box::pin(s2)
}
@@ -142,7 +141,7 @@ macro_rules! pipe1 {
pub async fn make_event_pipe(
evq: &RawEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error> {
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
if false {
match dbconn::channel_exists(&evq.channel, &node_config).await {
Ok(_) => (),
@@ -309,7 +308,7 @@ pub fn make_remote_event_blobs_stream(
pub async fn make_event_blobs_pipe(
evq: &RawEventsQuery,
node_config: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error> {
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
if false {
match dbconn::channel_exists(&evq.channel, &node_config).await {
Ok(_) => (),
@@ -331,9 +330,9 @@ pub async fn make_event_blobs_pipe(
evq.disk_io_tune.clone(),
node_config,
)?;
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable>);
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable + Send>);
//let s = tracing_futures::Instrumented::instrument(s, tracing::info_span!("make_event_blobs_pipe"));
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
pipe = Box::pin(s);
pipe
} else {
@@ -347,9 +346,9 @@ pub async fn make_event_blobs_pipe(
evq.disk_io_tune.clone(),
node_config,
)?;
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable>);
let s = event_blobs.map(|item| Box::new(item) as Box<dyn Framable + Send>);
//let s = tracing_futures::Instrumented::instrument(s, tracing::info_span!("make_event_blobs_pipe"));
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>;
let pipe: Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>;
pipe = Box::pin(s);
pipe
};

5
disk/src/raw/eventsfromframes.rs
View File

@@ -2,7 +2,7 @@ use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use futures_core::Stream;
use futures_util::StreamExt;
use items::frame::decode_frame;
use items::{FrameType, Sitemty, StreamItem};
use items::{FrameTypeStatic, Sitemty, StreamItem};
use netpod::log::*;
use serde::de::DeserializeOwned;
use std::marker::PhantomData;
@@ -37,8 +37,7 @@ where
impl<T, I> Stream for EventsFromFrames<T, I>
where
T: AsyncRead + Unpin,
I: DeserializeOwned + Unpin,
Sitemty<I>: FrameType,
I: FrameTypeStatic + DeserializeOwned + Unpin,
{
type Item = Sitemty<I>;