daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support more numeric types in conversion

Browse Source
This commit is contained in:
Dominik Werder
2021-12-16 15:40:53 +01:00
parent 7053af83b7
commit be666a932d
12 changed files with 282 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
dq/src/bin/daq-convert.rs
View File

@@ -1,6 +1,7 @@
use clap::{crate_version, Parser};
use err::Error;
use std::path::PathBuf;
use netpod::{timeunits::*, Nanos};
use std::{path::PathBuf, str::FromStr};
#[derive(Debug, Parser)]
#[clap(name="DAQ buffer tools", version=crate_version!())]
@@ -21,22 +22,52 @@ pub enum SubCmd {
pub struct ConvertArchiverApplianceChannel {
#[clap(
long,
about = "Prefix for keyspaces, e.g. specify `daq` to get scalar keyspace directory `daq_2`"
about = "Prefix for keyspaces, e.g. specify `daq` to get scalar keyspace directory `daq_2`."
)]
keyspace_prefix: String,
#[clap(long, about = "Name of the channel to convert")]
#[clap(long, about = "Name of the channel to convert.")]
channel_name: String,
#[clap(long, about = "Look for archiver appliance data at given path")]
#[clap(long, about = "Look for archiver appliance data at given path.")]
input_dir: PathBuf,
#[clap(long, about = "Generate Databuffer format at given path")]
#[clap(long, about = "Generate Databuffer format at given path.")]
output_dir: PathBuf,
#[clap(
default_value = "1d",
long,
about = "Size of the time-bins in the generated Databuffer format.\nUnit-suffixes: `h` (hours), `d` (days)"
)]
time_bin_size: TimeBinSize,
}
#[derive(Clone, Debug)]
pub struct TimeBinSize {
nanos: u64,
}
impl FromStr for TimeBinSize {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
if s.is_empty() {
return Err(Error::with_msg_no_trace("Malformed time-bin size"));
}
let suff = s.chars().last().unwrap();
if suff.is_numeric() {
Err(Error::with_msg_no_trace("Malformed time-bin size"))
} else if suff == 'h' {
let bs: u64 = s[..s.len() - 1].parse()?;
Ok(Self { nanos: bs * HOUR })
} else if suff == 'd' {
let bs: u64 = s[..s.len() - 1].parse()?;
Ok(Self { nanos: bs * DAY })
} else {
Err(Error::with_msg_no_trace("Malformed time-bin size"))
}
}
}
pub fn main() -> Result<(), Error> {
taskrun::run(async {
if false {
return Err(Error::with_msg_no_trace(format!("unknown command")));
}
let opts = Opts::parse();
match opts.subcmd {
SubCmd::ConvertArchiverApplianceChannel(sub) => {
@@ -45,6 +76,7 @@ pub fn main() -> Result<(), Error> {
channel_name: sub.channel_name,
input_dir: sub.input_dir,
output_dir: sub.output_dir,
time_bin_size: Nanos::from_ns(sub.time_bin_size.nanos),
};
dq::convert(params).await
}

26
dq/src/bin/dq.rs
View File

@@ -24,27 +24,10 @@ pub struct Opts {
#[derive(Debug, Parser)]
pub enum SubCmd {
#[clap(about = "Convert a channel from the Archiver Appliance into Databuffer format.")]
ConvertArchiverApplianceChannel(ConvertArchiverApplianceChannel),
ReadDatabufferConfigfile(ReadDatabufferConfigfile),
ReadDatabufferDatafile(ReadDatabufferDatafile),
}
#[derive(Debug, Parser)]
pub struct ConvertArchiverApplianceChannel {
#[clap(
long,
about = "Prefix for keyspaces, e.g. specify `daq` to get scalar keyspace directory `daq_2`"
)]
keyspace_prefix: String,
#[clap(long, about = "Name of the channel to convert")]
channel_name: String,
#[clap(long, about = "Look for archiver appliance data at given path")]
input_dir: PathBuf,
#[clap(long, about = "Generate Databuffer format at given path")]
output_dir: PathBuf,
}
#[derive(Debug, Parser)]
pub struct ReadDatabufferConfigfile {
#[clap(long)]
@@ -66,15 +49,6 @@ pub fn main() -> Result<(), Error> {
}
let opts = Opts::parse();
match opts.subcmd {
SubCmd::ConvertArchiverApplianceChannel(sub) => {
let params = dq::ConvertParams {
keyspace_prefix: sub.keyspace_prefix,
channel_name: sub.channel_name,
input_dir: sub.input_dir,
output_dir: sub.output_dir,
};
dq::convert(params).await
}
SubCmd::ReadDatabufferConfigfile(sub) => {
let mut file = File::open(&sub.configfile).await?;
let meta = file.metadata().await?;

28
dq/src/dq.rs
View File

@@ -111,6 +111,7 @@ impl DataWriter {
self.write_events(2, ScalarType::F64, &events.tss, &events.values)
.await?;
}
_ => todo!(),
},
PlainEvents::Wave(item) => match item {
WavePlainEvents::F64(_events) => {
@@ -319,6 +320,7 @@ pub struct ConvertParams {
pub channel_name: String,
pub input_dir: PathBuf,
pub output_dir: PathBuf,
pub time_bin_size: Nanos,
}
pub async fn convert(convert_params: ConvertParams) -> Result<(), Error> {
@@ -330,7 +332,7 @@ pub async fn convert(convert_params: ConvertParams) -> Result<(), Error> {
convert_params.output_dir
)));
}
let bs = Nanos::from_ns(DAY);
let bs = convert_params.time_bin_size.clone();
let mut channel_config: Option<Config> = None;
let channel = Channel {
backend: String::new(),
@@ -424,30 +426,6 @@ pub async fn convert(convert_params: ConvertParams) -> Result<(), Error> {
}
if channel_config.is_none() {
let ks = if ei.is_wave() { 3 } else { 2 };
let scalar_type_2 = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k {
PlainEvents::Scalar(k) => match k {
ScalarPlainEvents::U32(_) => ScalarType::U32,
ScalarPlainEvents::I8(_) => ScalarType::I8,
ScalarPlainEvents::I16(_) => ScalarType::I16,
ScalarPlainEvents::I32(_) => ScalarType::I32,
ScalarPlainEvents::F32(_) => ScalarType::F32,
ScalarPlainEvents::F64(_) => ScalarType::F64,
},
PlainEvents::Wave(k) => match k {
WavePlainEvents::I8(_) => ScalarType::I8,
WavePlainEvents::I16(_) => ScalarType::I16,
WavePlainEvents::I32(_) => ScalarType::I32,
WavePlainEvents::F32(_) => ScalarType::F32,
WavePlainEvents::F64(_) => ScalarType::F64,
},
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if scalar_type_2 != scalar_type {
let msg = format!("unexpected type: {:?} vs {:?}", scalar_type_2, scalar_type);
return Err(Error::with_msg_no_trace(msg));
}
let e = parse::channelconfig::ConfigEntry {
ts: 0,
pulse: 0,