daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on adding stats to the streams

Browse Source
This commit is contained in:
Dominik Werder
2021-05-01 18:00:05 +02:00
parent 0f9408e9f8
commit 0eb6b3364d
15 changed files with 275 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
disk/src/raw/conn.rs
View File

@@ -11,36 +11,30 @@ use futures_util::StreamExt;
#[allow(unused_imports)]
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{Node, NodeConfig, Shape};
use netpod::{NodeConfigCached, Shape};
use std::net::SocketAddr;
use tokio::io::AsyncWriteExt;
use tokio::net::tcp::OwnedWriteHalf;
use tokio::net::TcpStream;
use tracing::Instrument;
pub async fn raw_service(node_config: NodeConfig, node: Node) -> Result<(), Error> {
let addr = format!("{}:{}", node.listen, node.port_raw);
pub async fn raw_service(node_config: NodeConfigCached) -> Result<(), Error> {
let addr = format!("{}:{}", node_config.node.listen, node_config.node.port_raw);
let lis = tokio::net::TcpListener::bind(addr).await?;
loop {
match lis.accept().await {
Ok((stream, addr)) => {
let node = node.clone();
taskrun::spawn(raw_conn_handler(stream, addr, node_config.clone(), node));
taskrun::spawn(raw_conn_handler(stream, addr, node_config.clone()));
}
Err(e) => Err(e)?,
}
}
}
async fn raw_conn_handler(
stream: TcpStream,
addr: SocketAddr,
node_config: NodeConfig,
node: Node,
) -> Result<(), Error> {
async fn raw_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: NodeConfigCached) -> Result<(), Error> {
//use tracing_futures::Instrument;
let span1 = span!(Level::INFO, "raw::raw_conn_handler");
let r = raw_conn_handler_inner(stream, addr, &node_config, &node)
let r = raw_conn_handler_inner(stream, addr, &node_config)
.instrument(span1)
.await;
match r {
@@ -57,10 +51,9 @@ pub type RawConnOut = Result<MinMaxAvgScalarEventBatch, Error>;
async fn raw_conn_handler_inner(
stream: TcpStream,
addr: SocketAddr,
node_config: &NodeConfig,
node: &Node,
node_config: &NodeConfigCached,
) -> Result<(), Error> {
match raw_conn_handler_inner_try(stream, addr, node_config, node).await {
match raw_conn_handler_inner_try(stream, addr, node_config).await {
Ok(_) => (),
Err(mut ce) => {
/*error!(
@@ -94,8 +87,7 @@ impl<E: Into<Error>> From<(E, OwnedWriteHalf)> for ConnErr {
async fn raw_conn_handler_inner_try(
stream: TcpStream,
addr: SocketAddr,
node_config: &NodeConfig,
node: &Node,
node_config: &NodeConfigCached,
) -> Result<(), ConnErr> {
info!("raw_conn_handler SPAWNED for {:?}", addr);
let (netin, mut netout) = stream.into_split();
@@ -139,7 +131,7 @@ async fn raw_conn_handler_inner_try(
}
debug!("REQUEST {:?}", evq);
let range = &evq.range;
let channel_config = match read_local_config(&evq.channel, node).await {
let channel_config = match read_local_config(&evq.channel, &node_config.node).await {
Ok(k) => k,
Err(e) => return Err((e, netout))?,
};
@@ -180,16 +172,21 @@ async fn raw_conn_handler_inner_try(
buffer_size: 1024 * 4,
};
let buffer_size = 1024 * 4;
let mut s1 = EventBlobsComplete::new(range.clone(), query.channel_config.clone(), node.clone(), buffer_size)
.into_dim_1_f32_stream()
.into_binned_x_bins_1();
let mut s1 = EventBlobsComplete::new(
range.clone(),
query.channel_config.clone(),
node_config.node.clone(),
buffer_size,
)
.into_dim_1_f32_stream()
.into_binned_x_bins_1();
let mut e = 0;
while let Some(item) = s1.next().await {
if let Ok(k) = &item {
e += 1;
trace!(
"emit items sp {:2} e {:3} len {:3} {:10?} {:10?}",
node.split,
node_config.node.split,
e,
k.tss.len(),
k.tss.first().map(|k| k / SEC),