daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP change container type

Browse Source
This commit is contained in:
Dominik Werder
2024-11-30 11:44:49 +01:00
parent 927ef9ca55
commit aeefe566b7
5 changed files with 38 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/commonio/Cargo.toml
View File

@@ -14,7 +14,7 @@ bytes = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
chrono = "0.4"
async-channel = "1.9.0"
async-channel = "1.9"
parking_lot = "0.12"
crc32fast = "1.2"
daqbuf-err = { path = "../../../daqbuf-err" }

49
crates/disk/src/decode.rs
View File

@@ -8,6 +8,8 @@ use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::timebin::BinningggContainerEventsDyn;
use items_0::WithLen;
use items_2::binning::container_events::ContainerEvents;
use items_2::binning::container_events::EventValueType;
use items_2::empty::empty_events_dyn_ev;
use items_2::eventfull::EventFull;
use items_2::eventsdim0::EventsDim0;
@@ -16,6 +18,7 @@ use netpod::log::*;
use netpod::AggKind;
use netpod::ScalarType;
use netpod::Shape;
use netpod::TsNano;
use std::marker::PhantomData;
use std::mem;
use std::pin::Pin;
@@ -176,9 +179,9 @@ where
_m1: PhantomData<STY>,
}
impl<STY> ValueDim0FromBytesImpl<STY>
impl<EVT> ValueDim0FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
{
fn boxed() -> Box<dyn ValueFromBytes> {
Box::new(Self {
@@ -187,9 +190,9 @@ where
}
}
impl<STY> ValueDim0FromBytes for ValueDim0FromBytesImpl<STY>
impl<EVT> ValueDim0FromBytes for ValueDim0FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
{
fn convert(
&self,
@@ -199,11 +202,9 @@ where
endian: Endian,
events: &mut dyn BinningggContainerEventsDyn,
) -> Result<(), Error> {
if let Some(evs) = events.as_any_mut().downcast_mut::<EventsDim0<STY>>() {
let v = <STY as ScalarValueFromBytes<STY>>::convert(buf, endian)?;
evs.values.push_back(v);
evs.tss.push_back(ts);
evs.pulses.push_back(pulse);
if let Some(evs) = events.as_any_mut().downcast_mut::<ContainerEvents<EVT>>() {
let v = <EVT as ScalarValueFromBytes<EVT>>::convert(buf, endian)?;
evs.push_back(TsNano::from_ns(ts), v);
Ok(())
} else {
Err(Error::with_msg_no_trace("unexpected container"))
@@ -211,9 +212,9 @@ where
}
}
impl<STY> ValueFromBytes for ValueDim0FromBytesImpl<STY>
impl<EVT> ValueFromBytes for ValueDim0FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
{
fn convert(
&self,
@@ -235,9 +236,10 @@ where
_m1: PhantomData<STY>,
}
impl<STY> ValueDim1FromBytesImpl<STY>
impl<EVT> ValueDim1FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
Vec<EVT>: EventValueType,
{
fn boxed(shape: Shape) -> Box<dyn ValueFromBytes> {
Box::new(Self {
@@ -247,9 +249,10 @@ where
}
}
impl<STY> ValueFromBytes for ValueDim1FromBytesImpl<STY>
impl<EVT> ValueFromBytes for ValueDim1FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
Vec<EVT>: EventValueType,
{
fn convert(
&self,
@@ -263,9 +266,10 @@ where
}
}
impl<STY> ValueDim1FromBytes for ValueDim1FromBytesImpl<STY>
impl<EVT> ValueDim1FromBytes for ValueDim1FromBytesImpl<EVT>
where
STY: ScalarOps + ScalarValueFromBytes<STY>,
EVT: EventValueType + ScalarOps + ScalarValueFromBytes<EVT>,
Vec<EVT>: EventValueType,
{
fn convert(
&self,
@@ -275,16 +279,17 @@ where
endian: Endian,
events: &mut dyn BinningggContainerEventsDyn,
) -> Result<(), Error> {
if let Some(evs) = events.as_any_mut().downcast_mut::<EventsDim1<STY>>() {
if let Some(evs) = events.as_any_mut().downcast_mut::<ContainerEvents<Vec<EVT>>>() {
let n = if let Shape::Wave(n) = self.shape {
n
} else {
return Err(Error::with_msg_no_trace("ValueDim1FromBytesImpl bad shape"));
};
let v = <STY as ScalarValueFromBytes<STY>>::convert_dim1(buf, endian, n as _)?;
evs.values.push_back(v);
evs.tss.push_back(ts);
evs.pulses.push_back(pulse);
let v = <EVT as ScalarValueFromBytes<EVT>>::convert_dim1(buf, endian, n as _)?;
evs.push_back(TsNano::from_ns(ts), v);
// evs.values.push_back(v);
// evs.tss.push_back(ts);
// evs.pulses.push_back(pulse);
Ok(())
} else {
Err(Error::with_msg_no_trace("unexpected container"))

3
crates/nodenet/src/conn.rs
View File

@@ -94,6 +94,7 @@ async fn make_channel_events_stream_data(
scyqueue: Option<&ScyllaQueue>,
ncc: &NodeConfigCached,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<ChannelEvents>> + Send>>, Error> {
// ) -> Result<impl Stream<Item = Sitemty<ChannelEvents>>, Error> {
if subq.backend() == TEST_BACKEND {
let node_count = ncc.node_config.cluster.nodes.len() as u64;
let node_ix = ncc.ix as u64;
@@ -219,7 +220,7 @@ where
.await
{
match k {
Ok(StreamItem::DataItem(item)) => {
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item))) => {
frames.push(item);
}
Ok(item) => {

9
crates/nodenet/src/conn/test.rs
View File

@@ -4,6 +4,7 @@ use daqbuf_err as err;
use err::Error;
use futures_util::StreamExt;
use items_0::streamitem::sitem_data;
use items_0::streamitem::RangeCompletableItem;
use items_0::streamitem::Sitemty;
use items_0::streamitem::StreamItem;
use items_0::streamitem::ERROR_FRAME_TYPE_ID;
@@ -102,8 +103,8 @@ fn raw_data_00() {
let mut frames = InMemoryFrameStream::new(TcpReadAsBytes::new(netin), qu.inmem_bufcap());
while let Some(frame) = frames.next().await {
match frame {
Ok(frame) => match frame {
StreamItem::DataItem(k) => {
Ok(x) => match x {
StreamItem::DataItem(RangeCompletableItem::Data(k)) => {
eprintln!("{k:?}");
if k.tyid() == ITEMS_2_CHANNEL_EVENTS_FRAME_TYPE_ID {
} else if k.tyid() == ERROR_FRAME_TYPE_ID {
@@ -115,6 +116,10 @@ fn raw_data_00() {
let item: Sitemty<ChannelEvents> = decode_frame(&k).unwrap();
eprintln!("decoded: {:?}", item);
}
StreamItem::DataItem(RangeCompletableItem::RangeComplete) => {
eprintln!("decoded: RangeComplete");
todo!()
}
StreamItem::Log(_) => todo!(),
StreamItem::Stats(_) => todo!(),
},

3
crates/streamio/src/tcprawclient.rs
View File

@@ -87,8 +87,7 @@ where
let inp = Box::pin(inp) as BoxedBytesStream;
let frames = InMemoryFrameStream::new(inp, subq.inmem_bufcap());
let frames = frames.map_err(sitem_err2_from_string);
let frames = Box::pin(frames);
let stream = EventsFromFrames::<T>::new(frames, addr);
let stream = EventsFromFrames::<T, _>::new(frames, addr);
streams.push(Box::pin(stream) as _);
}
Ok(streams)