Refactor for easier channel conversion tool
This commit is contained in:
Dominik Werder
@@ -4,10 +4,13 @@ use crate::eventchunker::EventFull;
|
||||
use err::Error;
|
||||
use futures_core::Stream;
|
||||
use futures_util::StreamExt;
|
||||
use items::eventsitem::EventsItem;
|
||||
use items::eventvalues::EventValues;
|
||||
use items::numops::{BoolNum, NumOps};
|
||||
use items::plainevents::{PlainEvents, ScalarPlainEvents};
|
||||
use items::waveevents::{WaveEvents, WaveNBinner, WavePlainProc, WaveXBinner};
|
||||
use items::{Appendable, EventAppendable, EventsNodeProcessor, RangeCompletableItem, StreamItem};
|
||||
use items::{Appendable, EventAppendable, EventsNodeProcessor, RangeCompletableItem, Sitemty, StreamItem};
|
||||
use netpod::{ScalarType, Shape};
|
||||
use std::marker::PhantomData;
|
||||
use std::mem::size_of;
|
||||
use std::pin::Pin;
|
||||
@@ -300,3 +303,206 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct EventsItemStream {
|
||||
inp: Pin<Box<dyn Stream<Item = Sitemty<EventFull>>>>,
|
||||
done: bool,
|
||||
complete: bool,
|
||||
}
|
||||
|
||||
impl EventsItemStream {
|
||||
pub fn new(inp: Pin<Box<dyn Stream<Item = Sitemty<EventFull>>>>) -> Self {
|
||||
Self {
|
||||
inp,
|
||||
done: false,
|
||||
complete: false,
|
||||
}
|
||||
}
|
||||
|
||||
// TODO need some default expectation about the content type, because real world data does not
|
||||
// always contain that information per event, or even contains wrong information.
|
||||
fn decode(&mut self, ev: &EventFull) -> Result<Option<EventsItem>, Error> {
|
||||
// TODO define expected endian from parameters:
|
||||
let big_endian = false;
|
||||
// TODO:
|
||||
let mut tyi = None;
|
||||
let mut ret = None;
|
||||
for i1 in 0..ev.tss.len() {
|
||||
let ts = ev.tss[i1];
|
||||
let pulse = ev.pulses[i1];
|
||||
// TODO check that dtype, event endianness and event shape match our static
|
||||
// expectation about the data in this channel.
|
||||
let _ty = &ev.scalar_types[i1];
|
||||
let be = ev.be[i1];
|
||||
if be != big_endian {
|
||||
return Err(Error::with_msg(format!("big endian mismatch {} vs {}", be, big_endian)));
|
||||
}
|
||||
// TODO bad, data on disk is inconsistent, can not rely on endian as stated in channel config.
|
||||
let decomp = ev.decomp(i1);
|
||||
// If not done yet, infer the actual type from the (undocumented) combinations of channel
|
||||
// config parameters and values in the event data.
|
||||
// TODO
|
||||
match &tyi {
|
||||
Some(_) => {}
|
||||
None => {
|
||||
//let cont = EventValues::<f64>::empty();
|
||||
tyi = Some((ev.scalar_types[i1].clone(), ev.shapes[i1].clone()));
|
||||
match &tyi.as_ref().unwrap().1 {
|
||||
Shape::Scalar => match &tyi.as_ref().unwrap().0 {
|
||||
ScalarType::U8 => {
|
||||
// TODO
|
||||
let cont = EventValues::<i8>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
|
||||
}
|
||||
ScalarType::U16 => {
|
||||
// TODO
|
||||
let cont = EventValues::<i16>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I16(cont))));
|
||||
}
|
||||
ScalarType::U32 => {
|
||||
// TODO
|
||||
let cont = EventValues::<i32>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
|
||||
}
|
||||
ScalarType::U64 => {
|
||||
// TODO
|
||||
let cont = EventValues::<i32>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
|
||||
}
|
||||
ScalarType::I8 => {
|
||||
let cont = EventValues::<i8>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
|
||||
}
|
||||
ScalarType::I16 => {
|
||||
let cont = EventValues::<i16>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I16(cont))));
|
||||
}
|
||||
ScalarType::I32 => {
|
||||
let cont = EventValues::<i32>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
|
||||
}
|
||||
ScalarType::I64 => {
|
||||
// TODO
|
||||
let cont = EventValues::<i32>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
|
||||
}
|
||||
ScalarType::F32 => {
|
||||
let cont = EventValues::<f32>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::F32(cont))));
|
||||
}
|
||||
ScalarType::F64 => {
|
||||
let cont = EventValues::<f64>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::F64(cont))));
|
||||
}
|
||||
ScalarType::BOOL => {
|
||||
// TODO
|
||||
let cont = EventValues::<i8>::empty();
|
||||
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
|
||||
}
|
||||
},
|
||||
Shape::Wave(_) => todo!(),
|
||||
Shape::Image(..) => todo!(),
|
||||
}
|
||||
}
|
||||
};
|
||||
// TODO here, I expect that we found the type.
|
||||
let tyi = tyi.as_ref().unwrap();
|
||||
match &tyi.1 {
|
||||
Shape::Scalar => match &tyi.0 {
|
||||
ScalarType::U8 => todo!(),
|
||||
ScalarType::U16 => todo!(),
|
||||
ScalarType::U32 => todo!(),
|
||||
ScalarType::U64 => todo!(),
|
||||
ScalarType::I8 => todo!(),
|
||||
ScalarType::I16 => todo!(),
|
||||
ScalarType::I32 => todo!(),
|
||||
ScalarType::I64 => todo!(),
|
||||
ScalarType::F32 => todo!(),
|
||||
ScalarType::F64 => {
|
||||
let conv = EventValuesDim0Case::<f64>::new();
|
||||
let val = EventValueFromBytes::<_, LittleEndian>::convert(&conv, decomp, big_endian)?;
|
||||
match &mut ret {
|
||||
Some(ret) => match ret {
|
||||
EventsItem::Plain(ret) => match ret {
|
||||
PlainEvents::Scalar(ret) => match ret {
|
||||
ScalarPlainEvents::F64(ret) => {
|
||||
ret.tss.push(ts);
|
||||
// TODO
|
||||
let _ = pulse;
|
||||
ret.values.push(val);
|
||||
}
|
||||
_ => panic!(),
|
||||
},
|
||||
PlainEvents::Wave(_) => panic!(),
|
||||
},
|
||||
EventsItem::XBinnedEvents(_) => todo!(),
|
||||
},
|
||||
None => panic!(),
|
||||
}
|
||||
}
|
||||
ScalarType::BOOL => todo!(),
|
||||
},
|
||||
Shape::Wave(_) => todo!(),
|
||||
Shape::Image(_, _) => todo!(),
|
||||
}
|
||||
//let val = self.evs.convert(decomp, be)?;
|
||||
//let k = <<EVS as EventValueFromBytes<NTY, END>>::Batch as EventAppendable>::append_event(ret, ev.tss[i1], val);
|
||||
}
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for EventsItemStream {
|
||||
type Item = Sitemty<EventsItem>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
use Poll::*;
|
||||
loop {
|
||||
break if self.complete {
|
||||
panic!("poll_next on complete")
|
||||
} else if self.done {
|
||||
self.complete = true;
|
||||
Ready(None)
|
||||
} else {
|
||||
match self.inp.poll_next_unpin(cx) {
|
||||
Ready(item) => match item {
|
||||
Some(item) => match item {
|
||||
Ok(item) => match item {
|
||||
StreamItem::DataItem(item) => match item {
|
||||
RangeCompletableItem::RangeComplete => {
|
||||
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))))
|
||||
}
|
||||
RangeCompletableItem::Data(item) => match self.decode(&item) {
|
||||
Ok(res) => match res {
|
||||
Some(res) => {
|
||||
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(res)))))
|
||||
}
|
||||
None => {
|
||||
continue;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
self.done = true;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
},
|
||||
},
|
||||
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
|
||||
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
|
||||
},
|
||||
Err(e) => {
|
||||
self.done = true;
|
||||
Ready(Some(Err(e)))
|
||||
}
|
||||
},
|
||||
None => {
|
||||
self.done = true;
|
||||
Ready(None)
|
||||
}
|
||||
},
|
||||
Pending => Pending,
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -436,6 +436,7 @@ pub struct EventFull {
|
||||
pub pulses: Vec<u64>,
|
||||
pub blobs: Vec<Vec<u8>>,
|
||||
#[serde(serialize_with = "decomps_ser", deserialize_with = "decomps_de")]
|
||||
// TODO allow access to `decomps` via method which checks first if `blobs` is already the decomp.
|
||||
pub decomps: Vec<Option<BytesMut>>,
|
||||
pub scalar_types: Vec<ScalarType>,
|
||||
pub be: Vec<bool>,
|
||||
@@ -510,6 +511,13 @@ impl EventFull {
|
||||
self.shapes.push(shape);
|
||||
self.comps.push(comp);
|
||||
}
|
||||
|
||||
pub fn decomp(&self, i: usize) -> &[u8] {
|
||||
match &self.decomps[i] {
|
||||
Some(decomp) => &decomp,
|
||||
None => &self.blobs[i],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SitemtyFrameType for EventFull {
|
||||
|
||||
Reference in New Issue
Block a user