daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor transport frame maker

Browse Source
This commit is contained in:
Dominik Werder
2021-04-24 08:56:53 +02:00
parent 72d60b1fce
commit 18d5dba789
3 changed files with 135 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
disk/src/cache.rs
View File

@@ -1,7 +1,7 @@
use crate::agg::{IntoBinnedT, MinMaxAvgScalarBinBatch, MinMaxAvgScalarEventBatch}; use crate::agg::{IntoBinnedT, MinMaxAvgScalarBinBatch, MinMaxAvgScalarEventBatch};
use crate::merge::MergedMinMaxAvgScalarStream; use crate::merge::MergedMinMaxAvgScalarStream;
use crate::raw::{EventsQuery, InMemoryFrameAsyncReadStream}; use crate::raw::{EventsQuery, FrameType, InMemoryFrameAsyncReadStream};
use bytes::{BufMut, Bytes, BytesMut}; use bytes::Bytes;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use err::Error; use err::Error;
use futures_core::Stream; use futures_core::Stream;
@@ -144,24 +144,13 @@ impl Stream for BinnedBytesForHttpStream {
return Ready(None); return Ready(None);
} }
match self.inp.poll_next_unpin(cx) { match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => { Ready(Some(item)) => match super::raw::make_frame::<BinnedBytesForHttpStreamFrame>(&item) {
match bincode::serialize::<BinnedBytesForHttpStreamFrame>(&item) { Ok(buf) => Ready(Some(Ok(buf.freeze()))),
Ok(enc) => { Err(e) => {
// TODO optimize this... self.errored = true;
const HEAD: usize = super::raw::INMEM_FRAME_HEAD; Ready(Some(Err(e.into())))
let mut buf = BytesMut::with_capacity(enc.len() + HEAD);
buf.put_u32_le(super::raw::INMEM_FRAME_MAGIC);
buf.put_u32_le(enc.len() as u32);
buf.put_u32_le(0);
buf.put(enc.as_ref());
Ready(Some(Ok(buf.freeze())))
}
Err(e) => {
self.errored = true;
Ready(Some(Err(e.into())))
}
} }
} },
Ready(None) => { Ready(None) => {
self.completed = true; self.completed = true;
Ready(None) Ready(None)
@@ -241,24 +230,13 @@ impl Stream for PreBinnedValueByteStream {
return Ready(None); return Ready(None);
} }
match self.inp.poll_next_unpin(cx) { match self.inp.poll_next_unpin(cx) {
Ready(Some(item)) => { Ready(Some(item)) => match super::raw::make_frame::<PreBinnedHttpFrame>(&item) {
// TODO optimize this Ok(buf) => Ready(Some(Ok(buf.freeze()))),
const HEAD: usize = super::raw::INMEM_FRAME_HEAD; Err(e) => {
match bincode::serialize::<PreBinnedHttpFrame>(&item) { self.errored = true;
Ok(enc) => { Ready(Some(Err(e.into())))
let mut buf = BytesMut::with_capacity(enc.len() + HEAD);
buf.put_u32_le(super::raw::INMEM_FRAME_MAGIC);
buf.put_u32_le(enc.len() as u32);
buf.put_u32_le(0);
buf.put(enc.as_ref());
Ready(Some(Ok(buf.freeze())))
}
Err(e) => {
self.errored = true;
Ready(Some(Err(e.into())))
}
} }
} },
Ready(None) => Ready(None), Ready(None) => Ready(None),
Pending => Pending, Pending => Pending,
} }
@@ -474,6 +452,8 @@ impl PreBinnedValueFetchedStream {
} }
} }
// TODO use a newtype here to use a different FRAME_TYPE_ID compared to
// impl FrameType for BinnedBytesForHttpStreamFrame
pub type PreBinnedHttpFrame = Result<MinMaxAvgScalarBinBatch, Error>; pub type PreBinnedHttpFrame = Result<MinMaxAvgScalarBinBatch, Error>;
impl Stream for PreBinnedValueFetchedStream { impl Stream for PreBinnedValueFetchedStream {
@@ -486,10 +466,13 @@ impl Stream for PreBinnedValueFetchedStream {
break if let Some(res) = self.res.as_mut() { break if let Some(res) = self.res.as_mut() {
pin_mut!(res); pin_mut!(res);
match res.poll_next(cx) { match res.poll_next(cx) {
Ready(Some(Ok(buf))) => match bincode::deserialize::<PreBinnedHttpFrame>(&buf) { Ready(Some(Ok(frame))) => {
Ok(item) => Ready(Some(item)), assert!(frame.tyid() == <PreBinnedHttpFrame as FrameType>::FRAME_TYPE_ID);
Err(e) => Ready(Some(Err(e.into()))), match bincode::deserialize::<PreBinnedHttpFrame>(frame.buf()) {
}, Ok(item) => Ready(Some(item)),
Err(e) => Ready(Some(Err(e.into()))),
}
}
Ready(Some(Err(e))) => Ready(Some(Err(e.into()))), Ready(Some(Err(e))) => Ready(Some(Err(e.into()))),
Ready(None) => Ready(None), Ready(None) => Ready(None),
Pending => Pending, Pending => Pending,

178
disk/src/raw.rs
View File

@@ -6,6 +6,7 @@ to request such data from nodes.
*/ */
use crate::agg::MinMaxAvgScalarEventBatch; use crate::agg::MinMaxAvgScalarEventBatch;
use crate::cache::BinnedBytesForHttpStreamFrame;
use bytes::{BufMut, Bytes, BytesMut}; use bytes::{BufMut, Bytes, BytesMut};
use err::Error; use err::Error;
use futures_core::Stream; use futures_core::Stream;
@@ -33,23 +34,20 @@ pub struct EventsQuery {
pub agg_kind: AggKind, pub agg_kind: AggKind,
} }
#[derive(Serialize, Deserialize)]
pub struct EventQueryJsonStringFrame(String);
pub async fn x_processed_stream_from_node( pub async fn x_processed_stream_from_node(
query: Arc<EventsQuery>, query: Arc<EventsQuery>,
node: Arc<Node>, node: Arc<Node>,
) -> Result<Pin<Box<dyn Stream<Item = Result<MinMaxAvgScalarEventBatch, Error>> + Send>>, Error> { ) -> Result<Pin<Box<dyn Stream<Item = Result<MinMaxAvgScalarEventBatch, Error>> + Send>>, Error> {
let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?; let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
let qjs = serde_json::to_vec(query.as_ref())?; let qjs = serde_json::to_string(query.as_ref())?;
let (netin, mut netout) = net.into_split(); let (netin, mut netout) = net.into_split();
let buf = make_frame(&EventQueryJsonStringFrame(qjs))?;
// TODO this incorrect magic MUST bubble up into the final result and be reported. netout.write_all(&buf).await?;
let buf = make_term_frame();
netout.write_u32_le(INMEM_FRAME_MAGIC).await?; netout.write_all(&buf).await?;
netout.write_u32_le(qjs.len() as u32).await?;
netout.write_u32_le(0).await?;
netout.write_all(&qjs).await?;
netout.write_u32_le(INMEM_FRAME_MAGIC).await?;
netout.write_u32_le(0).await?;
netout.write_u32_le(0).await?;
netout.flush().await?; netout.flush().await?;
netout.forget(); netout.forget();
let frames = InMemoryFrameAsyncReadStream::new(netin); let frames = InMemoryFrameAsyncReadStream::new(netin);
@@ -86,21 +84,22 @@ where
let j = &mut self.inp; let j = &mut self.inp;
pin_mut!(j); pin_mut!(j);
break match j.poll_next(cx) { break match j.poll_next(cx) {
Ready(Some(Ok(buf))) => { Ready(Some(Ok(frame))) => {
type ExpectedType = RawConnOut;
info!( info!(
"MinMaxAvgScalarEventBatchStreamFromFrames got full frame buf {}", "MinMaxAvgScalarEventBatchStreamFromFrames got full frame buf {}",
buf.len() frame.buf().len()
); );
match bincode::deserialize::<RawConnOut>(buf.as_ref()) { assert!(frame.tyid() == <ExpectedType as FrameType>::FRAME_TYPE_ID);
match bincode::deserialize::<ExpectedType>(frame.buf()) {
Ok(item) => match item { Ok(item) => match item {
Ok(item) => Ready(Some(Ok(item))), Ok(item) => Ready(Some(Ok(item))),
Err(e) => Ready(Some(Err(e))), Err(e) => Ready(Some(Err(e))),
}, },
Err(e) => { Err(e) => {
trace!( trace!(
"MinMaxAvgScalarEventBatchStreamFromFrames ~~~~~~~~ ERROR on frame payload {} {}", "MinMaxAvgScalarEventBatchStreamFromFrames ~~~~~~~~ ERROR on frame payload {}",
buf.len(), frame.buf().len(),
crchex(buf)
); );
Ready(Some(Err(e.into()))) Ready(Some(Err(e.into())))
} }
@@ -114,7 +113,7 @@ where
} }
} }
pub const INMEM_FRAME_HEAD: usize = 12; pub const INMEM_FRAME_HEAD: usize = 16;
pub const INMEM_FRAME_MAGIC: u32 = 0xc6c3b73d; pub const INMEM_FRAME_MAGIC: u32 = 0xc6c3b73d;
/** /**
@@ -154,13 +153,14 @@ where
} }
} }
fn tryparse(&mut self) -> Option<Option<Result<Bytes, Error>>> { fn tryparse(&mut self) -> Option<Option<Result<InMemoryFrame, Error>>> {
const HEAD: usize = INMEM_FRAME_HEAD; const HEAD: usize = INMEM_FRAME_HEAD;
let mut buf = std::mem::replace(&mut self.buf, BytesMut::new()); let mut buf = std::mem::replace(&mut self.buf, BytesMut::new());
if self.wp >= HEAD { if self.wp >= HEAD {
let magic = u32::from_le_bytes(*arrayref::array_ref![buf, 0, 4]); let magic = u32::from_le_bytes(*arrayref::array_ref![buf, 0, 4]);
let len = u32::from_le_bytes(*arrayref::array_ref![buf, 4, 4]); let encid = u32::from_le_bytes(*arrayref::array_ref![buf, 4, 4]);
let _tyid = u32::from_le_bytes(*arrayref::array_ref![buf, 8, 4]); let tyid = u32::from_le_bytes(*arrayref::array_ref![buf, 8, 4]);
let len = u32::from_le_bytes(*arrayref::array_ref![buf, 12, 4]);
if magic != INMEM_FRAME_MAGIC { if magic != INMEM_FRAME_MAGIC {
error!("InMemoryFrameAsyncReadStream tryparse incorrect magic: {}", magic); error!("InMemoryFrameAsyncReadStream tryparse incorrect magic: {}", magic);
return Some(Some(Err(Error::with_msg(format!( return Some(Some(Err(Error::with_msg(format!(
@@ -206,7 +206,13 @@ where
self.wp = self.wp - nl; self.wp = self.wp - nl;
self.buf = buf3; self.buf = buf3;
self.inp_bytes_consumed += nl as u64; self.inp_bytes_consumed += nl as u64;
Some(Some(Ok(buf.freeze()))) let ret = InMemoryFrame {
len,
tyid,
encid,
buf: buf.freeze(),
};
Some(Some(Ok(ret)))
} else { } else {
self.buf = buf; self.buf = buf;
None None
@@ -219,11 +225,33 @@ where
} }
} }
pub struct InMemoryFrame {
encid: u32,
tyid: u32,
len: u32,
buf: Bytes,
}
impl InMemoryFrame {
pub fn encid(&self) -> u32 {
self.encid
}
pub fn tyid(&self) -> u32 {
self.tyid
}
pub fn len(&self) -> u32 {
self.len
}
pub fn buf(&self) -> &Bytes {
&self.buf
}
}
impl<T> Stream for InMemoryFrameAsyncReadStream<T> impl<T> Stream for InMemoryFrameAsyncReadStream<T>
where where
T: AsyncRead + Unpin, T: AsyncRead + Unpin,
{ {
type Item = Result<Bytes, Error>; type Item = Result<InMemoryFrame, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*; use Poll::*;
@@ -295,15 +323,52 @@ where
} }
} }
// TODO build a stream from disk data to batched event data. pub trait FrameType {
#[allow(dead_code)] const FRAME_TYPE_ID: u32;
async fn local_unpacked_test() { }
let query = err::todoval();
let node = err::todoval(); impl FrameType for BinnedBytesForHttpStreamFrame {
// TODO open and parse the channel config. const FRAME_TYPE_ID: u32 = 0x02;
// TODO find the matching config entry. (bonus: fuse consecutive compatible entries) }
use crate::agg::IntoDim1F32Stream;
let _stream = crate::EventBlobsComplete::new(&query, query.channel_config.clone(), node).into_dim_1_f32_stream(); impl FrameType for EventQueryJsonStringFrame {
const FRAME_TYPE_ID: u32 = 0x03;
}
impl FrameType for RawConnOut {
const FRAME_TYPE_ID: u32 = 0x04;
}
pub fn make_frame<FT>(item: &FT) -> Result<BytesMut, Error>
where
FT: FrameType + Serialize,
{
match bincode::serialize(item) {
Ok(enc) => {
if enc.len() > u32::MAX as usize {
return Err(Error::with_msg(format!("too long payload {}", enc.len())));
}
let encid = 0x12121212;
let mut buf = BytesMut::with_capacity(enc.len() + INMEM_FRAME_HEAD);
buf.put_u32_le(INMEM_FRAME_MAGIC);
buf.put_u32_le(encid);
buf.put_u32_le(FT::FRAME_TYPE_ID);
buf.put_u32_le(enc.len() as u32);
buf.put(enc.as_ref());
Ok(buf)
}
Err(e) => Err(e)?,
}
}
pub fn make_term_frame() -> BytesMut {
let encid = 0x12121313;
let mut buf = BytesMut::with_capacity(INMEM_FRAME_HEAD);
buf.put_u32_le(INMEM_FRAME_MAGIC);
buf.put_u32_le(encid);
buf.put_u32_le(0x01);
buf.put_u32_le(0);
buf
} }
pub async fn raw_service(node_config: Arc<NodeConfig>) -> Result<(), Error> { pub async fn raw_service(node_config: Arc<NodeConfig>) -> Result<(), Error> {
@@ -332,17 +397,7 @@ async fn raw_conn_handler_inner(stream: TcpStream, addr: SocketAddr) -> Result<(
Ok(_) => (), Ok(_) => (),
Err(mut ce) => { Err(mut ce) => {
error!("raw_conn_handler_inner CAUGHT ERROR AND TRY TO SEND OVER TCP"); error!("raw_conn_handler_inner CAUGHT ERROR AND TRY TO SEND OVER TCP");
let enc = bincode::serialize::<RawConnOut>(&Err(ce.err))?; let buf = make_frame::<RawConnOut>(&Err(ce.err))?;
// TODO optimize
let mut buf = BytesMut::with_capacity(enc.len() + INMEM_FRAME_HEAD);
buf.put_u32_le(INMEM_FRAME_MAGIC);
buf.put_u32_le(enc.len() as u32);
buf.put_u32_le(0);
buf.put(enc.as_ref());
trace!(
"raw_conn_handler_inner ~~~~~~~~~~~~ EMIT FRAME PAYLOAD CRC {}",
crchex(&enc)
);
match ce.netout.write(&buf).await { match ce.netout.write(&buf).await {
Ok(_) => (), Ok(_) => (),
Err(e) => return Err(e)?, Err(e) => return Err(e)?,
@@ -390,14 +445,12 @@ async fn raw_conn_handler_inner_try(stream: TcpStream, addr: SocketAddr) -> Resu
error!("expect a command frame"); error!("expect a command frame");
return Err((Error::with_msg("expect a command frame"), netout))?; return Err((Error::with_msg("expect a command frame"), netout))?;
} }
let s1 = match String::from_utf8(frames[0].to_vec()) { let qitem = match bincode::deserialize::<EventQueryJsonStringFrame>(frames[0].buf()) {
Ok(k) => k, Ok(k) => k,
Err(e) => { Err(e) => return Err((e, netout))?,
return Err((e, netout))?;
}
}; };
trace!("json {}", s1); trace!("json: {}", qitem.0);
let res: Result<serde_json::Value, _> = serde_json::from_str(&s1); let res: Result<serde_json::Value, _> = serde_json::from_str(&qitem.0);
let evq = match res { let evq = match res {
Ok(k) => k, Ok(k) => k,
Err(e) => { Err(e) => {
@@ -420,32 +473,17 @@ async fn raw_conn_handler_inner_try(stream: TcpStream, addr: SocketAddr) -> Resu
batch.avgs.push(9.6); batch.avgs.push(9.6);
let mut s1 = futures_util::stream::iter(vec![batch]).map(Result::Ok); let mut s1 = futures_util::stream::iter(vec![batch]).map(Result::Ok);
while let Some(item) = s1.next().await { while let Some(item) = s1.next().await {
match bincode::serialize::<RawConnOut>(&item) { match make_frame::<RawConnOut>(&item) {
Ok(enc) => { Ok(buf) => match netout.write(&buf).await {
let mut buf = BytesMut::with_capacity(enc.len() + INMEM_FRAME_HEAD); Ok(_) => {}
buf.put_u32_le(INMEM_FRAME_MAGIC); Err(e) => return Err((e, netout))?,
buf.put_u32_le(enc.len() as u32); },
buf.put_u32_le(0);
buf.put(enc.as_ref());
trace!(
"raw_conn_handler_inner_try ~~~~~~~~~~~~ EMIT FRAME PAYLOAD {} {}",
enc.len(),
crchex(enc)
);
match netout.write(&buf).await {
Ok(_) => {}
Err(e) => return Err((e, netout))?,
}
}
Err(e) => { Err(e) => {
return Err((e, netout))?; return Err((e, netout))?;
} }
} }
} }
let mut buf = BytesMut::with_capacity(32); let buf = make_term_frame();
buf.put_u32_le(INMEM_FRAME_MAGIC);
buf.put_u32_le(0);
buf.put_u32_le(0);
match netout.write(&buf).await { match netout.write(&buf).await {
Ok(_) => (), Ok(_) => (),
Err(e) => return Err((e, netout))?, Err(e) => return Err((e, netout))?,

7
retrieval/src/test.rs
View File

@@ -87,9 +87,10 @@ async fn get_cached_0_inner() -> Result<(), Error> {
.map_err(|e| error!("TEST GOT ERROR {:?}", e)) .map_err(|e| error!("TEST GOT ERROR {:?}", e))
.filter_map(|item| { .filter_map(|item| {
let g = match item { let g = match item {
Ok(buf) => { Ok(frame) => {
info!("TEST GOT FRAME len {}", buf.len()); type ExpectedType = disk::cache::BinnedBytesForHttpStreamFrame;
match bincode::deserialize::<disk::cache::BinnedBytesForHttpStreamFrame>(&buf) { info!("TEST GOT FRAME len {}", frame.buf().len());
match bincode::deserialize::<ExpectedType>(frame.buf()) {
Ok(item) => match item { Ok(item) => match item {
Ok(item) => { Ok(item) => {
info!("TEST GOT ITEM"); info!("TEST GOT ITEM");