daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on lots of changes

Browse Source
This commit is contained in:
Dominik Werder
2021-05-25 17:00:43 +02:00
parent d4c04496f6
commit 83adaa7321
20 changed files with 665 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
disk/src/raw/bffr.rs
View File

@@ -1,8 +1,7 @@
use crate::agg::eventbatch::MinMaxAvgScalarEventBatchStreamItem;
use crate::agg::streams::StreamItem;
use crate::binned::{BinnedStreamKind, RangeCompletableItem};
use crate::frame::inmem::InMemoryFrameAsyncReadStream;
use crate::frame::makeframe::decode_frame;
use crate::raw::conn::RawConnOut;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
@@ -11,39 +10,44 @@ use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncRead;
pub struct MinMaxAvgScalarEventBatchStreamFromFrames<T>
pub struct EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
{
inp: InMemoryFrameAsyncReadStream<T>,
errored: bool,
completed: bool,
stream_kind: SK,
}
impl<T> MinMaxAvgScalarEventBatchStreamFromFrames<T>
impl<T, SK> EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
{
pub fn new(inp: InMemoryFrameAsyncReadStream<T>) -> Self {
pub fn new(inp: InMemoryFrameAsyncReadStream<T>, stream_kind: SK) -> Self {
Self {
inp,
errored: false,
completed: false,
stream_kind,
}
}
}
impl<T> Stream for MinMaxAvgScalarEventBatchStreamFromFrames<T>
impl<T, SK> Stream for EventsFromFrames<T, SK>
where
T: AsyncRead + Unpin,
SK: BinnedStreamKind,
{
type Item = Result<StreamItem<MinMaxAvgScalarEventBatchStreamItem>, Error>;
type Item = Result<StreamItem<RangeCompletableItem<<SK as BinnedStreamKind>::XBinnedEvents>>, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.completed {
panic!("MinMaxAvgScalarEventBatchStreamFromFrames poll_next on completed");
panic!("EventsFromFrames poll_next on completed");
} else if self.errored {
self.completed = true;
Ready(None)
@@ -53,8 +57,7 @@ where
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
StreamItem::DataItem(frame) => {
type ExpectedType = RawConnOut;
match decode_frame::<ExpectedType>(&frame) {
match decode_frame::<<SK as BinnedStreamKind>::XBinnedEvents>(&frame) {
Ok(item) => match item {
Ok(item) => Ready(Some(Ok(item))),
Err(e) => {
@@ -64,7 +67,7 @@ where
},
Err(e) => {
error!(
"MinMaxAvgScalarEventBatchStreamFromFrames ~~~~~~~~ ERROR on frame payload {}",
"EventsFromFrames ~~~~~~~~ ERROR on frame payload {}",
frame.buf().len(),
);
self.errored = true;

64
disk/src/raw/conn.rs
View File

@@ -1,7 +1,8 @@
use crate::agg::binnedx::IntoBinnedXBins1;
use crate::agg::eventbatch::MinMaxAvgScalarEventBatchStreamItem;
use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
use crate::agg::streams::StreamItem;
use crate::agg::IntoDim1F32Stream;
use crate::binned::{BinnedStreamKind, BinnedStreamKindScalar, RangeCompletableItem};
use crate::channelconfig::{extract_matching_config_entry, read_local_config};
use crate::eventblobs::EventBlobsComplete;
use crate::eventchunker::EventChunkerConf;
@@ -11,30 +12,30 @@ use crate::raw::{EventQueryJsonStringFrame, EventsQuery};
use err::Error;
use futures_util::StreamExt;
use netpod::log::*;
use netpod::{ByteSize, NodeConfigCached, PerfOpts};
use netpod::{AggKind, ByteSize, NodeConfigCached, PerfOpts};
use std::net::SocketAddr;
use tokio::io::AsyncWriteExt;
use tokio::net::tcp::OwnedWriteHalf;
use tokio::net::TcpStream;
use tracing::Instrument;
pub async fn raw_service(node_config: NodeConfigCached) -> Result<(), Error> {
pub async fn events_service(node_config: NodeConfigCached) -> Result<(), Error> {
let addr = format!("{}:{}", node_config.node.listen, node_config.node.port_raw);
let lis = tokio::net::TcpListener::bind(addr).await?;
loop {
match lis.accept().await {
Ok((stream, addr)) => {
taskrun::spawn(raw_conn_handler(stream, addr, node_config.clone()));
taskrun::spawn(events_conn_handler(stream, addr, node_config.clone()));
}
Err(e) => Err(e)?,
}
}
}
async fn raw_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: NodeConfigCached) -> Result<(), Error> {
async fn events_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: NodeConfigCached) -> Result<(), Error> {
//use tracing_futures::Instrument;
let span1 = span!(Level::INFO, "raw::raw_conn_handler");
let r = raw_conn_handler_inner(stream, addr, &node_config)
let r = events_conn_handler_inner(stream, addr, &node_config)
.instrument(span1)
.await;
match r {
@@ -46,17 +47,16 @@ async fn raw_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: Node
}
}
pub type RawConnOut = Result<StreamItem<MinMaxAvgScalarEventBatchStreamItem>, Error>;
async fn raw_conn_handler_inner(
async fn events_conn_handler_inner(
stream: TcpStream,
addr: SocketAddr,
node_config: &NodeConfigCached,
) -> Result<(), Error> {
match raw_conn_handler_inner_try(stream, addr, node_config).await {
match events_conn_handler_inner_try(stream, addr, node_config).await {
Ok(_) => (),
Err(mut ce) => {
let buf = make_frame::<RawConnOut>(&Err(ce.err))?;
// TODO is it guaranteed to be compatible to serialize this way?
let buf = make_frame::<Result<StreamItem<MinMaxAvgScalarEventBatchStreamItem>, Error>>(&Err(ce.err))?;
match ce.netout.write_all(&buf).await {
Ok(_) => (),
Err(e) => return Err(e)?,
@@ -80,7 +80,7 @@ impl<E: Into<Error>> From<(E, OwnedWriteHalf)> for ConnErr {
}
}
async fn raw_conn_handler_inner_try(
async fn events_conn_handler_inner_try(
stream: TcpStream,
addr: SocketAddr,
node_config: &NodeConfigCached,
@@ -170,14 +170,40 @@ async fn raw_conn_handler_inner_try(
Ok(_) => {}
Err(_) => {}
}
match make_frame::<RawConnOut>(&item) {
Ok(buf) => match netout.write_all(&buf).await {
Ok(_) => {}
Err(e) => return Err((e, netout))?,
},
Err(e) => {
return Err((e, netout))?;
match evq.agg_kind {
AggKind::DimXBins1 => {
match make_frame::<
Result<
StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as BinnedStreamKind>::XBinnedEvents>>,
Error,
>,
>(&item)
{
Ok(buf) => match netout.write_all(&buf).await {
Ok(_) => {}
Err(e) => return Err((e, netout))?,
},
Err(e) => {
return Err((e, netout))?;
}
}
}
// TODO define this case:
AggKind::DimXBinsN(n1) => match make_frame::<
Result<
StreamItem<RangeCompletableItem<<BinnedStreamKindScalar as BinnedStreamKind>::XBinnedEvents>>,
Error,
>,
>(err::todoval())
{
Ok(buf) => match netout.write_all(&buf).await {
Ok(_) => {}
Err(e) => return Err((e, netout))?,
},
Err(e) => {
return Err((e, netout))?;
}
},
}
}
let buf = make_term_frame();