daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor for easier channel conversion tool

Browse Source
This commit is contained in:
Dominik Werder
2021-12-15 22:54:42 +01:00
parent 11229bd514
commit 7053af83b7
12 changed files with 1126 additions and 764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
archapp/src/archeng/datablock.rs
View File

@@ -451,21 +451,21 @@ pub async fn read_data2(
return Err(Error::with_msg_no_trace(format!("unexpected dbrcount {}", dbrcount)));
}
let res = match &dbrt {
DbrType::DbrTimeChar => read_msg!(i8, ex_s_i8, ex_v_i8, Byte, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeShort => read_msg!(i16, ex_s_i16, ex_v_i16, Short, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeLong => read_msg!(i32, ex_s_i32, ex_v_i32, Int, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeFloat => read_msg!(f32, ex_s_f32, ex_v_f32, Float, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeDouble => read_msg!(f64, ex_s_f64, ex_v_f64, Double, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeChar => read_msg!(i8, ex_s_i8, ex_v_i8, I8, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeShort => read_msg!(i16, ex_s_i16, ex_v_i16, I16, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeLong => read_msg!(i32, ex_s_i32, ex_v_i32, I32, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeFloat => read_msg!(f32, ex_s_f32, ex_v_f32, F32, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeDouble => read_msg!(f64, ex_s_f64, ex_v_f64, F64, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeString => {
if dbrcount == 1 {
// TODO
let evs = ScalarPlainEvents::Byte(EventValues::empty());
let evs = ScalarPlainEvents::I8(EventValues::empty());
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
// TODO
let evs = WavePlainEvents::Double(WaveEvents::empty());
let evs = WavePlainEvents::F64(WaveEvents::empty());
let plain = PlainEvents::Wave(evs);
let item = EventsItem::Plain(plain);
item
@@ -526,7 +526,7 @@ pub async fn read_data_1(
}
}
debug!("parsed block with {} / {} events", ntot, evs.tss.len());
let evs = ScalarPlainEvents::Double(evs);
let evs = ScalarPlainEvents::F64(evs);
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item

40
archapp/src/archeng/pipe.rs
View File

@@ -105,47 +105,47 @@ pub async fn make_event_pipe(
PlainEvents::Wave(j) => {
trace!("EventsItem::Plain Wave for {:?} {:?}", cfgshape, q_agg_kind);
match j {
WavePlainEvents::Byte(j) => {
WavePlainEvents::I8(j) => {
let binner =
WaveXBinner::<i8>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::Byte(out);
let item = SingleBinWaveEvents::I8(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Short(j) => {
WavePlainEvents::I16(j) => {
let binner =
WaveXBinner::<i16>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::Short(out);
let item = SingleBinWaveEvents::I16(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Int(j) => {
WavePlainEvents::I32(j) => {
let binner =
WaveXBinner::<i32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::Int(out);
let item = SingleBinWaveEvents::I32(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Float(j) => {
WavePlainEvents::F32(j) => {
let binner =
WaveXBinner::<f32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::Float(out);
let item = SingleBinWaveEvents::F32(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Double(j) => {
WavePlainEvents::F64(j) => {
let binner =
WaveXBinner::<f64>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::Double(out);
let item = SingleBinWaveEvents::F64(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
@@ -193,47 +193,47 @@ pub async fn make_event_pipe(
PlainEvents::Wave(j) => {
trace!("EventsItem::Plain Wave for {:?} {:?}", cfgshape, q_agg_kind);
match j {
WavePlainEvents::Byte(j) => {
WavePlainEvents::I8(j) => {
let binner =
WaveNBinner::<i8>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::Byte(out);
let item = MultiBinWaveEvents::I8(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Short(j) => {
WavePlainEvents::I16(j) => {
let binner =
WaveNBinner::<i16>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::Short(out);
let item = MultiBinWaveEvents::I16(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Int(j) => {
WavePlainEvents::I32(j) => {
let binner =
WaveNBinner::<i32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::Int(out);
let item = MultiBinWaveEvents::I32(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Float(j) => {
WavePlainEvents::F32(j) => {
let binner =
WaveNBinner::<f32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::Float(out);
let item = MultiBinWaveEvents::F32(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::Double(j) => {
WavePlainEvents::F64(j) => {
let binner =
WaveNBinner::<f64>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::Double(out);
let item = MultiBinWaveEvents::F64(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))

12
archapp/src/events.rs
View File

@@ -290,11 +290,11 @@ impl FrameMakerTrait for FrameMaker {
let shape = &self.shape;
let agg_kind = &self.agg_kind;
match scalar_type {
ScalarType::I8 => arm1!(item, i8, Byte, shape, agg_kind),
ScalarType::I16 => arm1!(item, i16, Short, shape, agg_kind),
ScalarType::I32 => arm1!(item, i32, Int, shape, agg_kind),
ScalarType::F32 => arm1!(item, f32, Float, shape, agg_kind),
ScalarType::F64 => arm1!(item, f64, Double, shape, agg_kind),
ScalarType::I8 => arm1!(item, i8, I8, shape, agg_kind),
ScalarType::I16 => arm1!(item, i16, I16, shape, agg_kind),
ScalarType::I32 => arm1!(item, i32, I32, shape, agg_kind),
ScalarType::F32 => arm1!(item, f32, F32, shape, agg_kind),
ScalarType::F64 => arm1!(item, f64, F64, shape, agg_kind),
_ => {
warn!("TODO for scalar_type {:?}", scalar_type);
err::todoval()
@@ -625,7 +625,7 @@ async fn linear_search_2(
#[allow(unused)]
fn events_item_to_framable(ei: EventsItem) -> Result<Box<dyn Framable + Send>, Error> {
match ei {
EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::Int(h))) => {
EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(h))) => {
let range: NanoRange = err::todoval();
let (x, y) = h
.tss

25
archapp/src/parse.rs
View File

@@ -50,7 +50,7 @@ fn parse_scalar_byte(m: &[u8], year: u32) -> Result<EventsItem, Error> {
let v = msg.get_val().first().map_or(0, |k| *k as i8);
t.tss.push(ts);
t.values.push(v);
Ok(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::Byte(t))))
Ok(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(t))))
}
macro_rules! scalar_parse {
@@ -65,6 +65,7 @@ macro_rules! scalar_parse {
let ts =
yd.timestamp() as u64 * 1000000000 + msg.get_secondsintoyear() as u64 * 1000000000 + msg.get_nano() as u64;
let v = msg.get_val();
//eprintln!("ts {} val {}", ts, v);
t.tss.push(ts);
t.values.push(v as $evty);
EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::$eit(t)))
@@ -175,37 +176,37 @@ impl PbFileReader {
let ei = match payload_type {
SCALAR_BYTE => parse_scalar_byte(m, year)?,
SCALAR_ENUM => {
scalar_parse!(m, year, ScalarEnum, Int, i32)
scalar_parse!(m, year, ScalarEnum, I32, i32)
}
SCALAR_SHORT => {
scalar_parse!(m, year, ScalarShort, Short, i16)
scalar_parse!(m, year, ScalarShort, I16, i16)
}
SCALAR_INT => {
scalar_parse!(m, year, ScalarInt, Int, i32)
scalar_parse!(m, year, ScalarInt, I32, i32)
}
SCALAR_FLOAT => {
scalar_parse!(m, year, ScalarFloat, Float, f32)
scalar_parse!(m, year, ScalarFloat, F32, f32)
}
SCALAR_DOUBLE => {
scalar_parse!(m, year, ScalarDouble, Double, f64)
scalar_parse!(m, year, ScalarDouble, F64, f64)
}
WAVEFORM_BYTE => {
wave_parse!(m, year, VectorChar, Byte, i8)
wave_parse!(m, year, VectorChar, I8, i8)
}
WAVEFORM_SHORT => {
wave_parse!(m, year, VectorShort, Short, i16)
wave_parse!(m, year, VectorShort, I16, i16)
}
WAVEFORM_ENUM => {
wave_parse!(m, year, VectorEnum, Int, i32)
wave_parse!(m, year, VectorEnum, I32, i32)
}
WAVEFORM_INT => {
wave_parse!(m, year, VectorInt, Int, i32)
wave_parse!(m, year, VectorInt, I32, i32)
}
WAVEFORM_FLOAT => {
wave_parse!(m, year, VectorFloat, Float, f32)
wave_parse!(m, year, VectorFloat, F32, f32)
}
WAVEFORM_DOUBLE => {
wave_parse!(m, year, VectorDouble, Double, f64)
wave_parse!(m, year, VectorDouble, F64, f64)
}
SCALAR_STRING | WAVEFORM_STRING | V4_GENERIC_BYTES => {
return Err(Error::with_msg_no_trace(format!("not supported: {:?}", payload_type)));

208
disk/src/decode.rs
View File

@@ -4,10 +4,13 @@ use crate::eventchunker::EventFull;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::eventvalues::EventValues;
use items::numops::{BoolNum, NumOps};
use items::plainevents::{PlainEvents, ScalarPlainEvents};
use items::waveevents::{WaveEvents, WaveNBinner, WavePlainProc, WaveXBinner};
use items::{Appendable, EventAppendable, EventsNodeProcessor, RangeCompletableItem, StreamItem};
use items::{Appendable, EventAppendable, EventsNodeProcessor, RangeCompletableItem, Sitemty, StreamItem};
use netpod::{ScalarType, Shape};
use std::marker::PhantomData;
use std::mem::size_of;
use std::pin::Pin;
@@ -300,3 +303,206 @@ where
}
}
}
pub struct EventsItemStream {
inp: Pin<Box<dyn Stream<Item = Sitemty<EventFull>>>>,
done: bool,
complete: bool,
}
impl EventsItemStream {
pub fn new(inp: Pin<Box<dyn Stream<Item = Sitemty<EventFull>>>>) -> Self {
Self {
inp,
done: false,
complete: false,
}
}
// TODO need some default expectation about the content type, because real world data does not
// always contain that information per event, or even contains wrong information.
fn decode(&mut self, ev: &EventFull) -> Result<Option<EventsItem>, Error> {
// TODO define expected endian from parameters:
let big_endian = false;
// TODO:
let mut tyi = None;
let mut ret = None;
for i1 in 0..ev.tss.len() {
let ts = ev.tss[i1];
let pulse = ev.pulses[i1];
// TODO check that dtype, event endianness and event shape match our static
// expectation about the data in this channel.
let _ty = &ev.scalar_types[i1];
let be = ev.be[i1];
if be != big_endian {
return Err(Error::with_msg(format!("big endian mismatch {} vs {}", be, big_endian)));
}
// TODO bad, data on disk is inconsistent, can not rely on endian as stated in channel config.
let decomp = ev.decomp(i1);
// If not done yet, infer the actual type from the (undocumented) combinations of channel
// config parameters and values in the event data.
// TODO
match &tyi {
Some(_) => {}
None => {
//let cont = EventValues::<f64>::empty();
tyi = Some((ev.scalar_types[i1].clone(), ev.shapes[i1].clone()));
match &tyi.as_ref().unwrap().1 {
Shape::Scalar => match &tyi.as_ref().unwrap().0 {
ScalarType::U8 => {
// TODO
let cont = EventValues::<i8>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
}
ScalarType::U16 => {
// TODO
let cont = EventValues::<i16>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I16(cont))));
}
ScalarType::U32 => {
// TODO
let cont = EventValues::<i32>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
}
ScalarType::U64 => {
// TODO
let cont = EventValues::<i32>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
}
ScalarType::I8 => {
let cont = EventValues::<i8>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
}
ScalarType::I16 => {
let cont = EventValues::<i16>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I16(cont))));
}
ScalarType::I32 => {
let cont = EventValues::<i32>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
}
ScalarType::I64 => {
// TODO
let cont = EventValues::<i32>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(cont))));
}
ScalarType::F32 => {
let cont = EventValues::<f32>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::F32(cont))));
}
ScalarType::F64 => {
let cont = EventValues::<f64>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::F64(cont))));
}
ScalarType::BOOL => {
// TODO
let cont = EventValues::<i8>::empty();
ret = Some(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(cont))));
}
},
Shape::Wave(_) => todo!(),
Shape::Image(..) => todo!(),
}
}
};
// TODO here, I expect that we found the type.
let tyi = tyi.as_ref().unwrap();
match &tyi.1 {
Shape::Scalar => match &tyi.0 {
ScalarType::U8 => todo!(),
ScalarType::U16 => todo!(),
ScalarType::U32 => todo!(),
ScalarType::U64 => todo!(),
ScalarType::I8 => todo!(),
ScalarType::I16 => todo!(),
ScalarType::I32 => todo!(),
ScalarType::I64 => todo!(),
ScalarType::F32 => todo!(),
ScalarType::F64 => {
let conv = EventValuesDim0Case::<f64>::new();
let val = EventValueFromBytes::<_, LittleEndian>::convert(&conv, decomp, big_endian)?;
match &mut ret {
Some(ret) => match ret {
EventsItem::Plain(ret) => match ret {
PlainEvents::Scalar(ret) => match ret {
ScalarPlainEvents::F64(ret) => {
ret.tss.push(ts);
// TODO
let _ = pulse;
ret.values.push(val);
}
_ => panic!(),
},
PlainEvents::Wave(_) => panic!(),
},
EventsItem::XBinnedEvents(_) => todo!(),
},
None => panic!(),
}
}
ScalarType::BOOL => todo!(),
},
Shape::Wave(_) => todo!(),
Shape::Image(_, _) => todo!(),
}
//let val = self.evs.convert(decomp, be)?;
//let k = <<EVS as EventValueFromBytes<NTY, END>>::Batch as EventAppendable>::append_event(ret, ev.tss[i1], val);
}
Ok(ret)
}
}
impl Stream for EventsItemStream {
type Item = Sitemty<EventsItem>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.complete {
panic!("poll_next on complete")
} else if self.done {
self.complete = true;
Ready(None)
} else {
match self.inp.poll_next_unpin(cx) {
Ready(item) => match item {
Some(item) => match item {
Ok(item) => match item {
StreamItem::DataItem(item) => match item {
RangeCompletableItem::RangeComplete => {
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))))
}
RangeCompletableItem::Data(item) => match self.decode(&item) {
Ok(res) => match res {
Some(res) => {
Ready(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(res)))))
}
None => {
continue;
}
},
Err(e) => {
self.done = true;
Ready(Some(Err(e)))
}
},
},
StreamItem::Log(item) => Ready(Some(Ok(StreamItem::Log(item)))),
StreamItem::Stats(item) => Ready(Some(Ok(StreamItem::Stats(item)))),
},
Err(e) => {
self.done = true;
Ready(Some(Err(e)))
}
},
None => {
self.done = true;
Ready(None)
}
},
Pending => Pending,
}
};
}
}
}

8
disk/src/eventchunker.rs
View File

@@ -436,6 +436,7 @@ pub struct EventFull {
pub pulses: Vec<u64>,
pub blobs: Vec<Vec<u8>>,
#[serde(serialize_with = "decomps_ser", deserialize_with = "decomps_de")]
// TODO allow access to `decomps` via method which checks first if `blobs` is already the decomp.
pub decomps: Vec<Option<BytesMut>>,
pub scalar_types: Vec<ScalarType>,
pub be: Vec<bool>,
@@ -510,6 +511,13 @@ impl EventFull {
self.shapes.push(shape);
self.comps.push(comp);
}
pub fn decomp(&self, i: usize) -> &[u8] {
match &self.decomps[i] {
Some(decomp) => &decomp,
None => &self.blobs[i],
}
}
}
impl SitemtyFrameType for EventFull {

53
dq/src/bin/daq-convert.rs Normal file
View File

@@ -0,0 +1,53 @@
use clap::{crate_version, Parser};
use err::Error;
use std::path::PathBuf;
#[derive(Debug, Parser)]
#[clap(name="DAQ buffer tools", version=crate_version!())]
pub struct Opts {
#[clap(short, long, parse(from_occurrences))]
pub verbose: u32,
#[clap(subcommand)]
pub subcmd: SubCmd,
}
#[derive(Debug, Parser)]
pub enum SubCmd {
#[clap(about = "Convert a channel from the Archiver Appliance into Databuffer format.")]
ConvertArchiverApplianceChannel(ConvertArchiverApplianceChannel),
}
#[derive(Debug, Parser)]
pub struct ConvertArchiverApplianceChannel {
#[clap(
long,
about = "Prefix for keyspaces, e.g. specify `daq` to get scalar keyspace directory `daq_2`"
)]
keyspace_prefix: String,
#[clap(long, about = "Name of the channel to convert")]
channel_name: String,
#[clap(long, about = "Look for archiver appliance data at given path")]
input_dir: PathBuf,
#[clap(long, about = "Generate Databuffer format at given path")]
output_dir: PathBuf,
}
pub fn main() -> Result<(), Error> {
taskrun::run(async {
if false {
return Err(Error::with_msg_no_trace(format!("unknown command")));
}
let opts = Opts::parse();
match opts.subcmd {
SubCmd::ConvertArchiverApplianceChannel(sub) => {
let params = dq::ConvertParams {
keyspace_prefix: sub.keyspace_prefix,
channel_name: sub.channel_name,
input_dir: sub.input_dir,
output_dir: sub.output_dir,
};
dq::convert(params).await
}
}
})
}

481
dq/src/bin/dq.rs
View File

@@ -1,23 +1,17 @@
use archapp::events::PositionState;
use archapp::parse::PbFileReader;
use bytes::BufMut;
use chrono::{TimeZone, Utc};
use clap::{crate_version, Parser};
//use disk::decode::EventValueShape;
//use disk::decode::EventValuesDim0Case;
use disk::eventchunker::EventChunkerConf;
use err::Error;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use netpod::query::RawEventsQuery;
use netpod::{log::*, ByteOrder, ByteSize, ChannelConfig, HasScalarType, HasShape};
use netpod::{timeunits::*, Shape};
use netpod::{AggKind, Channel, NanoRange, Nanos, ScalarType};
use parse::channelconfig::Config;
use std::io::SeekFrom;
use std::mem::take;
use netpod::log::*;
#[allow(unused)]
use netpod::timeunits::*;
use netpod::{ByteOrder, ByteSize, Channel, ChannelConfig, NanoRange, Shape};
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
#[derive(Debug, Parser)]
#[clap(name="DAQ buffer tools", version=crate_version!())]
@@ -65,264 +59,6 @@ pub struct ReadDatabufferDatafile {
datafile: PathBuf,
}
trait WritableValue {
fn put_value(&self, buf: &mut Vec<u8>);
}
impl WritableValue for f32 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_f32(*self);
}
}
impl WritableValue for f64 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_f64(*self);
}
}
struct DataWriter {
output_dir: PathBuf,
kspre: String,
channel: Channel,
bs: Nanos,
tb: u64,
datafile: Option<File>,
indexfile: Option<File>,
wpos: u64,
buf1: Vec<u8>,
}
impl DataWriter {
async fn new(output_dir: PathBuf, kspre: String, channel: Channel, bs: Nanos) -> Result<Self, Error> {
let ret = Self {
output_dir,
kspre,
channel,
bs,
tb: u64::MAX,
datafile: None,
indexfile: None,
wpos: 0,
buf1: vec![0; 1024 * 1024],
};
Ok(ret)
}
async fn write_item(&mut self, item: &PlainEvents) -> Result<(), Error> {
match item {
PlainEvents::Scalar(item) => match item {
ScalarPlainEvents::Float(events) => {
self.write_events(2, ScalarType::F32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::Double(events) => {
self.write_events(2, ScalarType::F64, &events.tss, &events.values)
.await?;
}
_ => todo!(),
},
PlainEvents::Wave(item) => match item {
WavePlainEvents::Double(_events) => {
todo!()
}
_ => todo!(),
},
}
Ok(())
}
async fn write_events<T: WritableValue>(
&mut self,
ks: u32,
scalar_type: ScalarType,
tss: &Vec<u64>,
vals: &Vec<T>,
) -> Result<(), Error> {
let split = 0;
assert_eq!(tss.len(), vals.len());
for i in 0..tss.len() {
let ts = tss[i];
let tb = ts / self.bs.ns;
if tb != self.tb {
let tbdate = chrono::Utc.timestamp((tb * (self.bs.ns / SEC)) as i64, 0);
eprintln!("Create directory for timebin {}", tbdate);
let p1 = self.output_dir.join(format!("{}_{}", self.kspre, ks));
let p2 = p1.join(self.channel.name());
let p3 = p2.join(format!("{:019}", tb));
let p4 = p3.join(format!("{:010}", split));
let p5 = p4.join(format!("{:019}_00000_Data", self.bs.ns / MS));
let p6 = p4.join(format!("{:019}_00000_Data_Index", self.bs.ns / MS));
tokio::fs::create_dir_all(&p4).await.map_err(|e| {
error!("Can not create {:?}", p4);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p5)
.await
.map_err(|e| {
error!("can not create new file {:?}", p5);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
file.write_all(&len1.to_be_bytes()).await?;
file.write_all(chs).await?;
file.write_all(&len1.to_be_bytes()).await?;
self.wpos = 10 + chs.len() as u64;
self.datafile = Some(file);
if ks == 3 {
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p6)
.await
.map_err(|e| {
error!("can not create new file {:?}", p6);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
self.indexfile = Some(file);
}
self.tb = tb;
}
let file = self.datafile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_i32(0);
buf.put_u64(0);
buf.put_u64(ts);
buf.put_u64(0);
buf.put_u64(0);
// Status, Severity
buf.put_u8(0);
buf.put_u8(0);
buf.put_i32(-1);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(scalar_type.index());
vals[i].put_value(&mut buf);
buf.put_i32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
self.buf1 = buf;
if ks == 3 {
let file = self.indexfile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_u64(ts);
buf.put_u64(self.wpos);
file.write_all(&buf).await?;
self.buf1 = buf;
}
self.wpos += len1 as u64;
}
Ok(())
}
async fn write_config(&mut self, config: &Config) -> Result<(), Error> {
eprintln!("Create directory for channel config");
let p1 = self.output_dir.join("config").join(self.channel.name()).join("latest");
tokio::fs::create_dir_all(&p1).await.map_err(|e| {
error!("Can not create {:?}", p1);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(p1.join("00000_Config"))
.await
.map_err(|e| {
error!("can not create config file in {:?}", p1);
e
})?;
let mut buf = take(&mut self.buf1);
{
buf.clear();
buf.put_u16(0);
file.write_all(&buf).await?;
}
{
buf.clear();
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
buf.put_u32(len1);
buf.put_slice(chs);
buf.put_u32(len1);
//let len1 = buf.len();
//buf[0..4].as_mut().put_u32(len1 as u32);
//buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
{
let e = &config.entries[0];
buf.clear();
buf.put_u32(0);
buf.put_u64(0);
buf.put_u64(0);
buf.put_i32(e.ks);
buf.put_u64(e.bs.ns / MS);
buf.put_i32(e.split_count);
buf.put_i32(e.status);
buf.put_i8(e.bb);
buf.put_i32(e.modulo);
buf.put_i32(e.offset);
buf.put_i16(e.precision);
let dtlen = 0;
buf.put_i32(dtlen);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(e.scalar_type.index());
if false {
// is shaped?
buf.put_u8(1);
buf.put_u32(16);
}
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_u32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
self.buf1 = buf;
Ok(())
}
}
impl Drop for DataWriter {
fn drop(&mut self) {
let indexfile = self.indexfile.take();
let datafile = self.datafile.take();
tokio::task::spawn(async move {
match indexfile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
match datafile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
});
}
}
pub fn main() -> Result<(), Error> {
taskrun::run(async {
if false {
@@ -331,189 +67,13 @@ pub fn main() -> Result<(), Error> {
let opts = Opts::parse();
match opts.subcmd {
SubCmd::ConvertArchiverApplianceChannel(sub) => {
let _ = tokio::fs::create_dir(&sub.output_dir).await;
let meta = tokio::fs::metadata(&sub.output_dir).await?;
if !meta.is_dir() {
return Err(Error::from_string(format!(
"Given output path is not a directory: {:?}",
sub.output_dir
)));
}
let bs = Nanos::from_ns(DAY);
let mut channel_config: Option<Config> = None;
let channel = Channel {
backend: String::new(),
name: sub.channel_name.into(),
let params = dq::ConvertParams {
keyspace_prefix: sub.keyspace_prefix,
channel_name: sub.channel_name,
input_dir: sub.input_dir,
output_dir: sub.output_dir,
};
let mut data_writer =
DataWriter::new(sub.output_dir, sub.keyspace_prefix.into(), channel.clone(), bs.clone()).await?;
let chandir = archapp::events::directory_for_channel_files(&channel, &sub.input_dir)?;
eprintln!("Looking for files in: {:?}", chandir);
let files = archapp::events::find_files_for_channel(&sub.input_dir, &channel).await?;
let mut evstot = 0;
for file in files {
eprintln!("Try to open {:?}", file);
let fni = archapp::events::parse_data_filename(file.to_str().unwrap())?;
debug!("fni: {:?}", fni);
let ts0 = Utc.ymd(fni.year as i32, fni.month, 1).and_hms(0, 0, 0);
let ts1 = ts0.timestamp() as u64 * SEC + ts0.timestamp_subsec_nanos() as u64;
let _ = ts1;
let mut f1 = File::open(&file).await?;
let _flen = f1.seek(SeekFrom::End(0)).await?;
f1.seek(SeekFrom::Start(0)).await?;
let pbr = PbFileReader::new(f1).await?;
debug!("channel name in pbr file: {:?}", pbr.channel_name());
debug!("data type in file: {:?}", pbr.payload_type());
let evq = RawEventsQuery {
channel: channel.clone(),
range: NanoRange {
beg: u64::MIN,
end: u64::MAX,
},
agg_kind: AggKind::Plain,
disk_io_buffer_size: 1024 * 4,
do_decompress: true,
};
let f1 = pbr.into_file();
// TODO can the positioning-logic maybe re-use the pbr?
let z = archapp::events::position_file_for_evq(f1, evq.clone(), fni.year).await?;
if let PositionState::Positioned(pos) = z.state {
let mut pbr = z.pbr;
assert_eq!(pos, pbr.abspos());
let mut i1 = 0;
let mut repnext = u64::MAX;
loop {
match pbr.read_msg().await {
Ok(Some(ei)) => {
use items::{WithLen, WithTimestamps};
let ei = ei.item;
if ei.is_wave() {
eprintln!("ERROR wave channels are not yet fully supported");
return Ok(());
}
if ei.len() > 0 {
let scalar_type = ei.scalar_type();
let shape = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k.shape() {
Shape::Scalar => None,
Shape::Wave(n) => Some(vec![n]),
Shape::Image(..) => panic!(),
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if let Some(conf) = &channel_config {
if scalar_type != conf.entries[0].scalar_type {
let msg = format!(
"unexpected type: {:?} vs {:?}",
scalar_type, conf.entries[0].scalar_type
);
return Err(Error::with_msg_no_trace(msg));
}
if shape != conf.entries[0].shape {
let msg = format!(
"unexpected shape: {:?} vs {:?}",
shape, conf.entries[0].shape
);
return Err(Error::with_msg_no_trace(msg));
}
}
if channel_config.is_none() {
let ks = if ei.is_wave() { 3 } else { 2 };
let scalar_type_2 = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k {
PlainEvents::Scalar(k) => match k {
ScalarPlainEvents::Byte(_) => ScalarType::I8,
ScalarPlainEvents::Short(_) => ScalarType::I16,
ScalarPlainEvents::Int(_) => ScalarType::I32,
ScalarPlainEvents::Float(_) => ScalarType::F32,
ScalarPlainEvents::Double(_) => ScalarType::F64,
},
PlainEvents::Wave(k) => match k {
WavePlainEvents::Byte(_) => ScalarType::I8,
WavePlainEvents::Short(_) => ScalarType::I16,
WavePlainEvents::Int(_) => ScalarType::I32,
WavePlainEvents::Float(_) => ScalarType::F32,
WavePlainEvents::Double(_) => ScalarType::F64,
},
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if scalar_type_2 != scalar_type {
let msg = format!(
"unexpected type: {:?} vs {:?}",
scalar_type_2, scalar_type
);
return Err(Error::with_msg_no_trace(msg));
}
let e = parse::channelconfig::ConfigEntry {
ts: 0,
pulse: 0,
ks,
bs: bs.clone(),
split_count: 1,
status: 0,
bb: 0,
modulo: 0,
offset: 0,
precision: 0,
scalar_type: scalar_type,
is_compressed: false,
is_shaped: false,
is_array: false,
byte_order: netpod::ByteOrder::LE,
compression_method: None,
shape,
source_name: None,
unit: None,
description: None,
optional_fields: None,
value_converter: None,
};
let k = parse::channelconfig::Config {
format_version: 0,
channel_name: channel.name().into(),
entries: vec![e],
};
channel_config = Some(k);
}
match &ei {
items::eventsitem::EventsItem::Plain(item) => {
data_writer.write_item(item).await?;
}
items::eventsitem::EventsItem::XBinnedEvents(_) => {
panic!()
}
}
}
let tslast = if ei.len() > 0 { Some(ei.ts(ei.len() - 1)) } else { None };
if i1 == repnext {
debug!("read msg from file {} len {} tslast {:?}", i1, ei.len(), tslast);
repnext = 1 + 4 * repnext / 3;
}
i1 += 1;
if false {
ei.x_aggregate(&evq.agg_kind);
}
}
Ok(None) => {
debug!("reached end of file");
break;
}
Err(e) => {
error!("error while reading msg {:?}", e);
break;
}
}
}
debug!("read total {} events from the last file", i1);
evstot += i1;
} else {
error!("Position fail.");
}
}
eprintln!("Total number of events converted: {}", evstot);
data_writer.write_config(channel_config.as_ref().unwrap()).await?;
Ok(())
dq::convert(params).await
}
SubCmd::ReadDatabufferConfigfile(sub) => {
let mut file = File::open(&sub.configfile).await?;
@@ -526,7 +86,6 @@ pub fn main() -> Result<(), Error> {
Err(e) => return Err(Error::with_msg_no_trace(format!("can not parse: {:?}", e))),
};
eprintln!("Read config: {:?}", config);
eprintln!("Config bs: {}", config.entries[0].bs.ns / MS);
Ok(())
}
SubCmd::ReadDatabufferDatafile(sub) => {
@@ -539,6 +98,7 @@ pub fn main() -> Result<(), Error> {
Ok(k) => k.1,
Err(e) => return Err(Error::with_msg_no_trace(format!("can not parse: {:?}", e))),
};
eprintln!("Read config: {:?}", config);
let file = File::open(&sub.datafile).await?;
let inp = Box::pin(disk::file_content_stream(
file,
@@ -566,7 +126,7 @@ pub fn main() -> Result<(), Error> {
disk_stats_every: ByteSize::mb(2),
};
let max_ts = Arc::new(AtomicU64::new(0));
let mut chunks = disk::eventchunker::EventChunker::from_start(
let chunks = disk::eventchunker::EventChunker::from_start(
inp,
channel_config.clone(),
range,
@@ -576,11 +136,11 @@ pub fn main() -> Result<(), Error> {
false,
true,
);
use futures_util::stream::StreamExt;
use items::WithLen;
while let Some(item) = chunks.next().await {
//let evs = EventValuesDim0Case::<f64>::new();
let mut stream = disk::decode::EventsItemStream::new(Box::pin(chunks));
while let Some(item) = stream.next().await {
let item = item?;
match item {
items::StreamItem::DataItem(item) => {
@@ -589,8 +149,7 @@ pub fn main() -> Result<(), Error> {
warn!("RangeComplete");
}
items::RangeCompletableItem::Data(item) => {
info!("Data len {}", item.len());
info!("{:?}", item);
info!("{:?} ({} events)", item, item.len());
}
};
}

519
dq/src/dq.rs
View File

@@ -1 +1,520 @@
use archapp::events::PositionState;
use archapp::parse::PbFileReader;
use bytes::BufMut;
use chrono::{TimeZone, Utc};
use err::Error;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::timeunits::*;
use netpod::{AggKind, Channel, HasScalarType, HasShape, NanoRange, Nanos, ScalarType, Shape};
use parse::channelconfig::Config;
use std::fmt;
use std::io::SeekFrom;
use std::mem::take;
use std::path::PathBuf;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
trait WritableValue: fmt::Debug {
fn put_value(&self, buf: &mut Vec<u8>);
}
impl WritableValue for u32 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_u32_le(*self);
}
}
impl WritableValue for i8 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_i8(*self);
}
}
impl WritableValue for i16 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_i16_le(*self);
}
}
impl WritableValue for i32 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_i32_le(*self);
}
}
impl WritableValue for f32 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_f32_le(*self);
}
}
impl WritableValue for f64 {
fn put_value(&self, buf: &mut Vec<u8>) {
buf.put_f64_le(*self);
}
}
struct DataWriter {
output_dir: PathBuf,
kspre: String,
channel: Channel,
bs: Nanos,
tb: u64,
datafile: Option<File>,
indexfile: Option<File>,
wpos: u64,
buf1: Vec<u8>,
}
impl DataWriter {
async fn new(output_dir: PathBuf, kspre: String, channel: Channel, bs: Nanos) -> Result<Self, Error> {
let ret = Self {
output_dir,
kspre,
channel,
bs,
tb: u64::MAX,
datafile: None,
indexfile: None,
wpos: 0,
buf1: vec![0; 1024 * 1024],
};
Ok(ret)
}
async fn write_item(&mut self, item: &PlainEvents) -> Result<(), Error> {
match item {
PlainEvents::Scalar(item) => match item {
ScalarPlainEvents::U32(events) => {
self.write_events(2, ScalarType::U32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I8(events) => {
self.write_events(2, ScalarType::I8, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I16(events) => {
self.write_events(2, ScalarType::I16, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I32(events) => {
self.write_events(2, ScalarType::I32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::F32(events) => {
self.write_events(2, ScalarType::F32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::F64(events) => {
self.write_events(2, ScalarType::F64, &events.tss, &events.values)
.await?;
}
},
PlainEvents::Wave(item) => match item {
WavePlainEvents::F64(_events) => {
todo!()
}
_ => todo!(),
},
}
Ok(())
}
async fn write_events<T: WritableValue>(
&mut self,
ks: u32,
scalar_type: ScalarType,
tss: &Vec<u64>,
vals: &Vec<T>,
) -> Result<(), Error> {
let split = 0;
assert_eq!(tss.len(), vals.len());
for i in 0..tss.len() {
let ts = tss[i];
let tb = ts / self.bs.ns;
if tb != self.tb {
let tbdate = chrono::Utc.timestamp((tb * (self.bs.ns / SEC)) as i64, 0);
eprintln!("Create directory for timebin {}", tbdate);
let p1 = self.output_dir.join(format!("{}_{}", self.kspre, ks));
let p2 = p1.join(self.channel.name());
let p3 = p2.join(format!("{:019}", tb));
let p4 = p3.join(format!("{:010}", split));
let p5 = p4.join(format!("{:019}_00000_Data", self.bs.ns / MS));
let p6 = p4.join(format!("{:019}_00000_Data_Index", self.bs.ns / MS));
tokio::fs::create_dir_all(&p4).await.map_err(|e| {
error!("Can not create {:?}", p4);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p5)
.await
.map_err(|e| {
error!("can not create new file {:?}", p5);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
file.write_all(&len1.to_be_bytes()).await?;
file.write_all(chs).await?;
file.write_all(&len1.to_be_bytes()).await?;
self.wpos = 10 + chs.len() as u64;
self.datafile = Some(file);
if ks == 3 {
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p6)
.await
.map_err(|e| {
error!("can not create new file {:?}", p6);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
self.indexfile = Some(file);
}
self.tb = tb;
}
let file = self.datafile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_i32(0);
buf.put_u64(0);
buf.put_u64(ts);
buf.put_u64(0);
buf.put_u64(0);
// Status, Severity
buf.put_u8(0);
buf.put_u8(0);
buf.put_i32(-1);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(scalar_type.index());
vals[i].put_value(&mut buf);
buf.put_i32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
self.buf1 = buf;
if ks == 3 {
let file = self.indexfile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_u64(ts);
buf.put_u64(self.wpos);
file.write_all(&buf).await?;
self.buf1 = buf;
}
self.wpos += len1 as u64;
}
Ok(())
}
async fn write_config(&mut self, config: &Config) -> Result<(), Error> {
eprintln!("Create directory for channel config");
let p1 = self.output_dir.join("config").join(self.channel.name()).join("latest");
tokio::fs::create_dir_all(&p1).await.map_err(|e| {
error!("Can not create {:?}", p1);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(p1.join("00000_Config"))
.await
.map_err(|e| {
error!("can not create config file in {:?}", p1);
e
})?;
let mut buf = take(&mut self.buf1);
{
buf.clear();
buf.put_u16(0);
file.write_all(&buf).await?;
}
{
buf.clear();
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
buf.put_u32(len1);
buf.put_slice(chs);
buf.put_u32(len1);
//let len1 = buf.len();
//buf[0..4].as_mut().put_u32(len1 as u32);
//buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
{
let e = &config.entries[0];
buf.clear();
buf.put_u32(0);
buf.put_u64(0);
buf.put_u64(0);
buf.put_i32(e.ks);
buf.put_u64(e.bs.ns / MS);
buf.put_i32(e.split_count);
buf.put_i32(e.status);
buf.put_i8(e.bb);
buf.put_i32(e.modulo);
buf.put_i32(e.offset);
buf.put_i16(e.precision);
let dtlen = 0;
buf.put_i32(dtlen);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(e.scalar_type.index());
if false {
// is shaped?
buf.put_u8(1);
buf.put_u32(16);
}
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_u32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
self.buf1 = buf;
Ok(())
}
}
impl Drop for DataWriter {
fn drop(&mut self) {
let indexfile = self.indexfile.take();
let datafile = self.datafile.take();
tokio::task::spawn(async move {
match indexfile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
match datafile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
});
}
}
#[derive(Clone, Debug)]
pub struct ConvertParams {
pub keyspace_prefix: String,
pub channel_name: String,
pub input_dir: PathBuf,
pub output_dir: PathBuf,
}
pub async fn convert(convert_params: ConvertParams) -> Result<(), Error> {
let _ = tokio::fs::create_dir(&convert_params.output_dir).await;
let meta = tokio::fs::metadata(&convert_params.output_dir).await?;
if !meta.is_dir() {
return Err(Error::from_string(format!(
"Given output path is not a directory: {:?}",
convert_params.output_dir
)));
}
let bs = Nanos::from_ns(DAY);
let mut channel_config: Option<Config> = None;
let channel = Channel {
backend: String::new(),
name: convert_params.channel_name.into(),
};
let mut data_writer = DataWriter::new(
convert_params.output_dir,
convert_params.keyspace_prefix.into(),
channel.clone(),
bs.clone(),
)
.await?;
let chandir = archapp::events::directory_for_channel_files(&channel, &convert_params.input_dir)?;
eprintln!("Looking for files in: {:?}", chandir);
let files = archapp::events::find_files_for_channel(&convert_params.input_dir, &channel).await?;
let mut evstot = 0;
for file in files {
eprintln!("Try to open {:?}", file);
let fni = archapp::events::parse_data_filename(file.to_str().unwrap())?;
debug!("fni: {:?}", fni);
let ts0 = Utc.ymd(fni.year as i32, fni.month, 1).and_hms(0, 0, 0);
let ts1 = ts0.timestamp() as u64 * SEC + ts0.timestamp_subsec_nanos() as u64;
let _ = ts1;
let mut f1 = File::open(&file).await?;
let _flen = f1.seek(SeekFrom::End(0)).await?;
f1.seek(SeekFrom::Start(0)).await?;
let pbr = PbFileReader::new(f1).await?;
eprintln!(
"PBR file header channel name: {:?} data type: {:?}",
pbr.channel_name(),
pbr.payload_type()
);
debug!("channel name in pbr file: {:?}", pbr.channel_name());
debug!("data type in file: {:?}", pbr.payload_type());
if pbr.channel_name() != channel.name() {
return Err(Error::with_msg(format!(
"channel name mismatch: {:?} vs {:?}",
pbr.channel_name(),
channel.name()
)));
}
let evq = RawEventsQuery {
channel: channel.clone(),
range: NanoRange {
beg: u64::MIN,
end: u64::MAX,
},
agg_kind: AggKind::Plain,
disk_io_buffer_size: 1024 * 4,
do_decompress: true,
};
let f1 = pbr.into_file();
// TODO can the positioning-logic maybe re-use the pbr?
let z = archapp::events::position_file_for_evq(f1, evq.clone(), fni.year).await?;
if let PositionState::Positioned(pos) = z.state {
let mut pbr = z.pbr;
assert_eq!(pos, pbr.abspos());
let mut i1 = 0;
let mut repnext = u64::MAX;
loop {
match pbr.read_msg().await {
Ok(Some(ei)) => {
use items::{WithLen, WithTimestamps};
let ei = ei.item;
if ei.is_wave() {
eprintln!("ERROR wave channels are not yet fully supported");
return Ok(());
}
if ei.len() > 0 {
let scalar_type = ei.scalar_type();
let shape = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k.shape() {
Shape::Scalar => None,
Shape::Wave(n) => Some(vec![n]),
Shape::Image(..) => panic!(),
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if let Some(conf) = &channel_config {
if scalar_type != conf.entries[0].scalar_type {
let msg = format!(
"unexpected type: {:?} vs {:?}",
scalar_type, conf.entries[0].scalar_type
);
return Err(Error::with_msg_no_trace(msg));
}
if shape != conf.entries[0].shape {
let msg = format!("unexpected shape: {:?} vs {:?}", shape, conf.entries[0].shape);
return Err(Error::with_msg_no_trace(msg));
}
}
if channel_config.is_none() {
let ks = if ei.is_wave() { 3 } else { 2 };
let scalar_type_2 = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k {
PlainEvents::Scalar(k) => match k {
ScalarPlainEvents::U32(_) => ScalarType::U32,
ScalarPlainEvents::I8(_) => ScalarType::I8,
ScalarPlainEvents::I16(_) => ScalarType::I16,
ScalarPlainEvents::I32(_) => ScalarType::I32,
ScalarPlainEvents::F32(_) => ScalarType::F32,
ScalarPlainEvents::F64(_) => ScalarType::F64,
},
PlainEvents::Wave(k) => match k {
WavePlainEvents::I8(_) => ScalarType::I8,
WavePlainEvents::I16(_) => ScalarType::I16,
WavePlainEvents::I32(_) => ScalarType::I32,
WavePlainEvents::F32(_) => ScalarType::F32,
WavePlainEvents::F64(_) => ScalarType::F64,
},
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if scalar_type_2 != scalar_type {
let msg = format!("unexpected type: {:?} vs {:?}", scalar_type_2, scalar_type);
return Err(Error::with_msg_no_trace(msg));
}
let e = parse::channelconfig::ConfigEntry {
ts: 0,
pulse: 0,
ks,
bs: bs.clone(),
split_count: 1,
status: 0,
bb: 0,
modulo: 0,
offset: 0,
precision: 0,
scalar_type: scalar_type,
is_compressed: false,
is_shaped: false,
is_array: false,
byte_order: netpod::ByteOrder::LE,
compression_method: None,
shape,
source_name: None,
unit: None,
description: None,
optional_fields: None,
value_converter: None,
};
let k = parse::channelconfig::Config {
format_version: 0,
channel_name: channel.name().into(),
entries: vec![e],
};
channel_config = Some(k);
}
match &ei {
items::eventsitem::EventsItem::Plain(item) => {
data_writer.write_item(item).await?;
}
items::eventsitem::EventsItem::XBinnedEvents(_) => {
panic!()
}
}
}
let tslast = if ei.len() > 0 { Some(ei.ts(ei.len() - 1)) } else { None };
if i1 == repnext {
debug!("read msg from file {} len {} tslast {:?}", i1, ei.len(), tslast);
repnext = 1 + 4 * repnext / 3;
}
i1 += 1;
if false {
ei.x_aggregate(&evq.agg_kind);
}
}
Ok(None) => {
debug!("reached end of file");
break;
}
Err(e) => {
error!("error while reading msg {:?}", e);
break;
}
}
}
debug!("read total {} events from the last file", i1);
evstot += i1;
} else {
error!("Position fail.");
}
}
eprintln!("Total number of events converted: {}", evstot);
data_writer.write_config(channel_config.as_ref().unwrap()).await?;
Ok(())
}

248
items/src/binnedevents.rs
View File

@@ -11,31 +11,31 @@ use crate::{
#[derive(Debug, Serialize, Deserialize)]
pub enum SingleBinWaveEvents {
Byte(XBinnedScalarEvents<i8>),
Short(XBinnedScalarEvents<i16>),
Int(XBinnedScalarEvents<i32>),
Float(XBinnedScalarEvents<f32>),
Double(XBinnedScalarEvents<f64>),
I8(XBinnedScalarEvents<i8>),
I16(XBinnedScalarEvents<i16>),
I32(XBinnedScalarEvents<i32>),
F32(XBinnedScalarEvents<f32>),
F64(XBinnedScalarEvents<f64>),
}
impl SingleBinWaveEvents {
pub fn variant_name(&self) -> String {
use SingleBinWaveEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
I8(_) => format!("I8"),
I16(_) => format!("I16"),
I32(_) => format!("I32"),
F32(_) => format!("F32"),
F64(_) => format!("F64"),
}
}
fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use SingleBinWaveEvents::*;
match self {
Byte(k) => match ak {
I8(k) => match ak {
AggKind::EventBlobs => panic!(),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::Byte(k))),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::I8(k))),
AggKind::TimeWeightedScalar => err::todoval(),
AggKind::DimXBins1 => err::todoval(),
AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())),
@@ -48,11 +48,11 @@ impl SingleBinWaveEvents {
impl Clearable for SingleBinWaveEvents {
fn clear(&mut self) {
match self {
SingleBinWaveEvents::Byte(k) => k.clear(),
SingleBinWaveEvents::Short(k) => k.clear(),
SingleBinWaveEvents::Int(k) => k.clear(),
SingleBinWaveEvents::Float(k) => k.clear(),
SingleBinWaveEvents::Double(k) => k.clear(),
SingleBinWaveEvents::I8(k) => k.clear(),
SingleBinWaveEvents::I16(k) => k.clear(),
SingleBinWaveEvents::I32(k) => k.clear(),
SingleBinWaveEvents::F32(k) => k.clear(),
SingleBinWaveEvents::F64(k) => k.clear(),
}
}
}
@@ -60,34 +60,34 @@ impl Clearable for SingleBinWaveEvents {
impl Appendable for SingleBinWaveEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
Self::I8(k) => Self::I8(k.empty_like_self()),
Self::I16(k) => Self::I16(k.empty_like_self()),
Self::I32(k) => Self::I32(k.empty_like_self()),
Self::F32(k) => Self::F32(k.empty_like_self()),
Self::F64(k) => Self::F64(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
Self::I8(k) => match src {
Self::I8(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
Self::I16(k) => match src {
Self::I16(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
Self::I32(k) => match src {
Self::I32(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
Self::F32(k) => match src {
Self::F32(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
Self::F64(k) => match src {
Self::F64(j) => k.append(j),
_ => panic!(),
},
}
@@ -97,24 +97,24 @@ impl Appendable for SingleBinWaveEvents {
impl PushableIndex for SingleBinWaveEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
Self::I8(k) => match src {
Self::I8(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
Self::I16(k) => match src {
Self::I16(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
Self::I32(k) => match src {
Self::I32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
Self::F32(k) => match src {
Self::F32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
Self::F64(k) => match src {
Self::F64(j) => k.push_index(j, ix),
_ => panic!(),
},
}
@@ -125,11 +125,11 @@ impl WithLen for SingleBinWaveEvents {
fn len(&self) -> usize {
use SingleBinWaveEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
I8(j) => j.len(),
I16(j) => j.len(),
I32(j) => j.len(),
F32(j) => j.len(),
F64(j) => j.len(),
}
}
}
@@ -138,11 +138,11 @@ impl WithTimestamps for SingleBinWaveEvents {
fn ts(&self, ix: usize) -> u64 {
use SingleBinWaveEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
I8(j) => j.ts(ix),
I16(j) => j.ts(ix),
I32(j) => j.ts(ix),
F32(j) => j.ts(ix),
F64(j) => j.ts(ix),
}
}
}
@@ -151,11 +151,11 @@ impl HasShape for SingleBinWaveEvents {
fn shape(&self) -> Shape {
use SingleBinWaveEvents::*;
match self {
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
I8(_) => Shape::Scalar,
I16(_) => Shape::Scalar,
I32(_) => Shape::Scalar,
F32(_) => Shape::Scalar,
F64(_) => Shape::Scalar,
}
}
}
@@ -164,42 +164,42 @@ impl HasScalarType for SingleBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use SingleBinWaveEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
I8(_) => ScalarType::I8,
I16(_) => ScalarType::I16,
I32(_) => ScalarType::I32,
F32(_) => ScalarType::F32,
F64(_) => ScalarType::F64,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum MultiBinWaveEvents {
Byte(XBinnedWaveEvents<i8>),
Short(XBinnedWaveEvents<i16>),
Int(XBinnedWaveEvents<i32>),
Float(XBinnedWaveEvents<f32>),
Double(XBinnedWaveEvents<f64>),
I8(XBinnedWaveEvents<i8>),
I16(XBinnedWaveEvents<i16>),
I32(XBinnedWaveEvents<i32>),
F32(XBinnedWaveEvents<f32>),
F64(XBinnedWaveEvents<f64>),
}
impl MultiBinWaveEvents {
pub fn variant_name(&self) -> String {
use MultiBinWaveEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
I8(_) => format!("I8"),
I16(_) => format!("I16"),
I32(_) => format!("I32"),
F32(_) => format!("F32"),
F64(_) => format!("F64"),
}
}
fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use MultiBinWaveEvents::*;
match self {
Byte(k) => match ak {
I8(k) => match ak {
AggKind::EventBlobs => panic!(),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::MultiBinWave(MultiBinWaveEvents::Byte(k))),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::MultiBinWave(MultiBinWaveEvents::I8(k))),
AggKind::TimeWeightedScalar => err::todoval(),
AggKind::DimXBins1 => err::todoval(),
AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())),
@@ -212,11 +212,11 @@ impl MultiBinWaveEvents {
impl Clearable for MultiBinWaveEvents {
fn clear(&mut self) {
match self {
MultiBinWaveEvents::Byte(k) => k.clear(),
MultiBinWaveEvents::Short(k) => k.clear(),
MultiBinWaveEvents::Int(k) => k.clear(),
MultiBinWaveEvents::Float(k) => k.clear(),
MultiBinWaveEvents::Double(k) => k.clear(),
MultiBinWaveEvents::I8(k) => k.clear(),
MultiBinWaveEvents::I16(k) => k.clear(),
MultiBinWaveEvents::I32(k) => k.clear(),
MultiBinWaveEvents::F32(k) => k.clear(),
MultiBinWaveEvents::F64(k) => k.clear(),
}
}
}
@@ -224,34 +224,34 @@ impl Clearable for MultiBinWaveEvents {
impl Appendable for MultiBinWaveEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
Self::I8(k) => Self::I8(k.empty_like_self()),
Self::I16(k) => Self::I16(k.empty_like_self()),
Self::I32(k) => Self::I32(k.empty_like_self()),
Self::F32(k) => Self::F32(k.empty_like_self()),
Self::F64(k) => Self::F64(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
Self::I8(k) => match src {
Self::I8(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
Self::I16(k) => match src {
Self::I16(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
Self::I32(k) => match src {
Self::I32(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
Self::F32(k) => match src {
Self::F32(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
Self::F64(k) => match src {
Self::F64(j) => k.append(j),
_ => panic!(),
},
}
@@ -261,24 +261,24 @@ impl Appendable for MultiBinWaveEvents {
impl PushableIndex for MultiBinWaveEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
Self::I8(k) => match src {
Self::I8(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
Self::I16(k) => match src {
Self::I16(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
Self::I32(k) => match src {
Self::I32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
Self::F32(k) => match src {
Self::F32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
Self::F64(k) => match src {
Self::F64(j) => k.push_index(j, ix),
_ => panic!(),
},
}
@@ -289,11 +289,11 @@ impl WithLen for MultiBinWaveEvents {
fn len(&self) -> usize {
use MultiBinWaveEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
I8(j) => j.len(),
I16(j) => j.len(),
I32(j) => j.len(),
F32(j) => j.len(),
F64(j) => j.len(),
}
}
}
@@ -302,11 +302,11 @@ impl WithTimestamps for MultiBinWaveEvents {
fn ts(&self, ix: usize) -> u64 {
use MultiBinWaveEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
I8(j) => j.ts(ix),
I16(j) => j.ts(ix),
I32(j) => j.ts(ix),
F32(j) => j.ts(ix),
F64(j) => j.ts(ix),
}
}
}
@@ -315,11 +315,11 @@ impl HasShape for MultiBinWaveEvents {
fn shape(&self) -> Shape {
use MultiBinWaveEvents::*;
match self {
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
I8(_) => Shape::Scalar,
I16(_) => Shape::Scalar,
I32(_) => Shape::Scalar,
F32(_) => Shape::Scalar,
F64(_) => Shape::Scalar,
}
}
}
@@ -328,11 +328,11 @@ impl HasScalarType for MultiBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use MultiBinWaveEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
I8(_) => ScalarType::I8,
I16(_) => ScalarType::I16,
I32(_) => ScalarType::I32,
F32(_) => ScalarType::F32,
F64(_) => ScalarType::F64,
}
}
}

21
items/src/eventsitem.rs
View File

@@ -49,21 +49,22 @@ impl EventsItem {
match self {
EventsItem::Plain(k) => match k {
PlainEvents::Scalar(k) => match k {
ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar),
ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar),
ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar),
ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar),
ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar),
ScalarPlainEvents::U32(_) => (ScalarType::U32, Shape::Scalar),
ScalarPlainEvents::I8(_) => (ScalarType::I8, Shape::Scalar),
ScalarPlainEvents::I16(_) => (ScalarType::I16, Shape::Scalar),
ScalarPlainEvents::I32(_) => (ScalarType::I32, Shape::Scalar),
ScalarPlainEvents::F32(_) => (ScalarType::F32, Shape::Scalar),
ScalarPlainEvents::F64(_) => (ScalarType::F64, Shape::Scalar),
},
PlainEvents::Wave(k) => match k {
// TODO
// Inherent issue for the non-static-type backends:
// there is a chance that we can't determine the shape here.
WavePlainEvents::Byte(k) => (ScalarType::I8, k.shape().unwrap()),
WavePlainEvents::Short(k) => (ScalarType::I16, k.shape().unwrap()),
WavePlainEvents::Int(k) => (ScalarType::I32, k.shape().unwrap()),
WavePlainEvents::Float(k) => (ScalarType::F32, k.shape().unwrap()),
WavePlainEvents::Double(k) => (ScalarType::F64, k.shape().unwrap()),
WavePlainEvents::I8(k) => (ScalarType::I8, k.shape().unwrap()),
WavePlainEvents::I16(k) => (ScalarType::I16, k.shape().unwrap()),
WavePlainEvents::I32(k) => (ScalarType::I32, k.shape().unwrap()),
WavePlainEvents::F32(k) => (ScalarType::F32, k.shape().unwrap()),
WavePlainEvents::F64(k) => (ScalarType::F64, k.shape().unwrap()),
},
},
EventsItem::XBinnedEvents(_k) => panic!(),

259
items/src/plainevents.rs
View File

@@ -9,22 +9,24 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize)]
pub enum ScalarPlainEvents {
Byte(EventValues<i8>),
Short(EventValues<i16>),
Int(EventValues<i32>),
Float(EventValues<f32>),
Double(EventValues<f64>),
U32(EventValues<u32>),
I8(EventValues<i8>),
I16(EventValues<i16>),
I32(EventValues<i32>),
F32(EventValues<f32>),
F64(EventValues<f64>),
}
impl ScalarPlainEvents {
pub fn variant_name(&self) -> String {
use ScalarPlainEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
U32(_) => format!("U32"),
I8(_) => format!("I8"),
I16(_) => format!("I16"),
I32(_) => format!("I32"),
F32(_) => format!("F32"),
F64(_) => format!("F64"),
}
}
}
@@ -32,11 +34,12 @@ impl ScalarPlainEvents {
impl Clearable for ScalarPlainEvents {
fn clear(&mut self) {
match self {
ScalarPlainEvents::Byte(k) => k.clear(),
ScalarPlainEvents::Short(k) => k.clear(),
ScalarPlainEvents::Int(k) => k.clear(),
ScalarPlainEvents::Float(k) => k.clear(),
ScalarPlainEvents::Double(k) => k.clear(),
ScalarPlainEvents::U32(k) => k.clear(),
ScalarPlainEvents::I8(k) => k.clear(),
ScalarPlainEvents::I16(k) => k.clear(),
ScalarPlainEvents::I32(k) => k.clear(),
ScalarPlainEvents::F32(k) => k.clear(),
ScalarPlainEvents::F64(k) => k.clear(),
}
}
}
@@ -44,34 +47,39 @@ impl Clearable for ScalarPlainEvents {
impl Appendable for ScalarPlainEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
Self::U32(k) => Self::U32(k.empty_like_self()),
Self::I8(k) => Self::I8(k.empty_like_self()),
Self::I16(k) => Self::I16(k.empty_like_self()),
Self::I32(k) => Self::I32(k.empty_like_self()),
Self::F32(k) => Self::F32(k.empty_like_self()),
Self::F64(k) => Self::F64(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
Self::U32(k) => match src {
Self::U32(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
Self::I8(k) => match src {
Self::I8(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
Self::I16(k) => match src {
Self::I16(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
Self::I32(k) => match src {
Self::I32(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
Self::F32(k) => match src {
Self::F32(j) => k.append(j),
_ => panic!(),
},
Self::F64(k) => match src {
Self::F64(j) => k.append(j),
_ => panic!(),
},
}
@@ -81,24 +89,28 @@ impl Appendable for ScalarPlainEvents {
impl PushableIndex for ScalarPlainEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
Self::U32(k) => match src {
Self::U32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
Self::I8(k) => match src {
Self::I8(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
Self::I16(k) => match src {
Self::I16(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
Self::I32(k) => match src {
Self::I32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
Self::F32(k) => match src {
Self::F32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::F64(k) => match src {
Self::F64(j) => k.push_index(j, ix),
_ => panic!(),
},
}
@@ -109,11 +121,12 @@ impl WithLen for ScalarPlainEvents {
fn len(&self) -> usize {
use ScalarPlainEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
U32(j) => j.len(),
I8(j) => j.len(),
I16(j) => j.len(),
I32(j) => j.len(),
F32(j) => j.len(),
F64(j) => j.len(),
}
}
}
@@ -122,11 +135,12 @@ impl WithTimestamps for ScalarPlainEvents {
fn ts(&self, ix: usize) -> u64 {
use ScalarPlainEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
U32(j) => j.ts(ix),
I8(j) => j.ts(ix),
I16(j) => j.ts(ix),
I32(j) => j.ts(ix),
F32(j) => j.ts(ix),
F64(j) => j.ts(ix),
}
}
}
@@ -143,32 +157,33 @@ impl HasScalarType for ScalarPlainEvents {
fn scalar_type(&self) -> ScalarType {
use ScalarPlainEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
U32(_) => ScalarType::U32,
I8(_) => ScalarType::I8,
I16(_) => ScalarType::I16,
I32(_) => ScalarType::I32,
F32(_) => ScalarType::F32,
F64(_) => ScalarType::F64,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum WavePlainEvents {
Byte(WaveEvents<i8>),
Short(WaveEvents<i16>),
Int(WaveEvents<i32>),
Float(WaveEvents<f32>),
Double(WaveEvents<f64>),
I8(WaveEvents<i8>),
I16(WaveEvents<i16>),
I32(WaveEvents<i32>),
F32(WaveEvents<f32>),
F64(WaveEvents<f64>),
}
impl WavePlainEvents {
pub fn shape(&self) -> Result<Shape, Error> {
match self {
WavePlainEvents::Byte(k) => k.shape(),
WavePlainEvents::Short(k) => k.shape(),
WavePlainEvents::Int(k) => k.shape(),
WavePlainEvents::Float(k) => k.shape(),
WavePlainEvents::Double(k) => k.shape(),
WavePlainEvents::I8(k) => k.shape(),
WavePlainEvents::I16(k) => k.shape(),
WavePlainEvents::I32(k) => k.shape(),
WavePlainEvents::F32(k) => k.shape(),
WavePlainEvents::F64(k) => k.shape(),
}
}
}
@@ -197,11 +212,11 @@ impl WavePlainEvents {
pub fn variant_name(&self) -> String {
use WavePlainEvents::*;
match self {
Byte(h) => format!("Byte({})", h.vals.first().map_or(0, |j| j.len())),
Short(h) => format!("Short({})", h.vals.first().map_or(0, |j| j.len())),
Int(h) => format!("Int({})", h.vals.first().map_or(0, |j| j.len())),
Float(h) => format!("Float({})", h.vals.first().map_or(0, |j| j.len())),
Double(h) => format!("Double({})", h.vals.first().map_or(0, |j| j.len())),
I8(h) => format!("I8({})", h.vals.first().map_or(0, |j| j.len())),
I16(h) => format!("I16({})", h.vals.first().map_or(0, |j| j.len())),
I32(h) => format!("I32({})", h.vals.first().map_or(0, |j| j.len())),
F32(h) => format!("F32({})", h.vals.first().map_or(0, |j| j.len())),
F64(h) => format!("F64({})", h.vals.first().map_or(0, |j| j.len())),
}
}
@@ -209,11 +224,11 @@ impl WavePlainEvents {
use WavePlainEvents::*;
let shape = self.shape().unwrap();
match self {
Byte(k) => wagg1!(k, ak, shape, Byte),
Short(k) => wagg1!(k, ak, shape, Short),
Int(k) => wagg1!(k, ak, shape, Int),
Float(k) => wagg1!(k, ak, shape, Float),
Double(k) => wagg1!(k, ak, shape, Double),
I8(k) => wagg1!(k, ak, shape, I8),
I16(k) => wagg1!(k, ak, shape, I16),
I32(k) => wagg1!(k, ak, shape, I32),
F32(k) => wagg1!(k, ak, shape, F32),
F64(k) => wagg1!(k, ak, shape, F64),
}
}
}
@@ -221,11 +236,11 @@ impl WavePlainEvents {
impl Clearable for WavePlainEvents {
fn clear(&mut self) {
match self {
WavePlainEvents::Byte(k) => k.clear(),
WavePlainEvents::Short(k) => k.clear(),
WavePlainEvents::Int(k) => k.clear(),
WavePlainEvents::Float(k) => k.clear(),
WavePlainEvents::Double(k) => k.clear(),
WavePlainEvents::I8(k) => k.clear(),
WavePlainEvents::I16(k) => k.clear(),
WavePlainEvents::I32(k) => k.clear(),
WavePlainEvents::F32(k) => k.clear(),
WavePlainEvents::F64(k) => k.clear(),
}
}
}
@@ -233,34 +248,34 @@ impl Clearable for WavePlainEvents {
impl Appendable for WavePlainEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
Self::I8(k) => Self::I8(k.empty_like_self()),
Self::I16(k) => Self::I16(k.empty_like_self()),
Self::I32(k) => Self::I32(k.empty_like_self()),
Self::F32(k) => Self::F32(k.empty_like_self()),
Self::F64(k) => Self::F64(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
Self::I8(k) => match src {
Self::I8(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
Self::I16(k) => match src {
Self::I16(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
Self::I32(k) => match src {
Self::I32(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
Self::F32(k) => match src {
Self::F32(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
Self::F64(k) => match src {
Self::F64(j) => k.append(j),
_ => panic!(),
},
}
@@ -270,24 +285,24 @@ impl Appendable for WavePlainEvents {
impl PushableIndex for WavePlainEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
Self::I8(k) => match src {
Self::I8(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
Self::I16(k) => match src {
Self::I16(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
Self::I32(k) => match src {
Self::I32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
Self::F32(k) => match src {
Self::F32(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
Self::F64(k) => match src {
Self::F64(j) => k.push_index(j, ix),
_ => panic!(),
},
}
@@ -298,11 +313,11 @@ impl WithLen for WavePlainEvents {
fn len(&self) -> usize {
use WavePlainEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
I8(j) => j.len(),
I16(j) => j.len(),
I32(j) => j.len(),
F32(j) => j.len(),
F64(j) => j.len(),
}
}
}
@@ -311,11 +326,11 @@ impl WithTimestamps for WavePlainEvents {
fn ts(&self, ix: usize) -> u64 {
use WavePlainEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
I8(j) => j.ts(ix),
I16(j) => j.ts(ix),
I32(j) => j.ts(ix),
F32(j) => j.ts(ix),
F64(j) => j.ts(ix),
}
}
}
@@ -325,8 +340,8 @@ impl HasShape for WavePlainEvents {
/*use WavePlainEvents::*;
match self {
Byte(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Short(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Int(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
I16(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
I32(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Float(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Double(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
}*/
@@ -338,11 +353,11 @@ impl HasScalarType for WavePlainEvents {
fn scalar_type(&self) -> ScalarType {
use WavePlainEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
I8(_) => ScalarType::I8,
I16(_) => ScalarType::I16,
I32(_) => ScalarType::I32,
F32(_) => ScalarType::F32,
F64(_) => ScalarType::F64,
}
}
}