Move workspace crates into subfolder

crates/dq/src/bin/daq-convert.rs

@@ -0,0 +1,79 @@
+use clap::{ArgAction, Parser};
+use err::Error;
+use netpod::timeunits::*;
+use std::path::PathBuf;
+use std::str::FromStr;
+
+#[derive(Debug, Parser)]
+#[command(author, version)]
+pub struct Opts {
+    #[arg(short, long, action(ArgAction::Count))]
+    pub verbose: u32,
+    #[command(subcommand)]
+    pub subcmd: SubCmd,
+}
+
+#[derive(Debug, Parser)]
+pub enum SubCmd {
+    /// Convert a channel from the Archiver Appliance into Databuffer format.
+    ConvertArchiverApplianceChannel(ConvertArchiverApplianceChannel),
+}
+
+#[derive(Debug, Parser)]
+pub struct ConvertArchiverApplianceChannel {
+    /// Prefix for keyspaces, e.g. specify `daq` to get scalar keyspace directory `daq_2`.
+    #[arg(long)]
+    keyspace_prefix: String,
+    /// Name of the channel to convert.
+    #[arg(long)]
+    channel_name: String,
+    /// Look for archiver appliance data at given path.
+    #[arg(long)]
+    input_dir: PathBuf,
+    /// Generate Databuffer format at given path.
+    #[arg(long)]
+    output_dir: PathBuf,
+    /// Size of the time-bins in the generated Databuffer format.\nUnit-suffixes: `h` (hours), `d` (days)
+    #[arg(default_value = "1d", long)]
+    time_bin_size: TimeBinSize,
+}
+
+#[derive(Clone, Debug)]
+pub struct TimeBinSize {
+    #[allow(unused)]
+    nanos: u64,
+}
+
+impl FromStr for TimeBinSize {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s.is_empty() {
+            return Err(Error::with_msg_no_trace("Malformed time-bin size"));
+        }
+        let suff = s.chars().last().unwrap();
+        if suff.is_numeric() {
+            Err(Error::with_msg_no_trace("Malformed time-bin size"))
+        } else if suff == 'h' {
+            let bs: u64 = s[..s.len() - 1].parse()?;
+            Ok(Self { nanos: bs * HOUR })
+        } else if suff == 'd' {
+            let bs: u64 = s[..s.len() - 1].parse()?;
+            Ok(Self { nanos: bs * DAY })
+        } else {
+            Err(Error::with_msg_no_trace("Malformed time-bin size"))
+        }
+    }
+}
+
+pub fn main() -> Result<(), Error> {
+    taskrun::run(async {
+        let opts = Opts::parse();
+        match opts.subcmd {
+            SubCmd::ConvertArchiverApplianceChannel(_) => {
+                eprintln!("error: archapp not built");
+                Ok(())
+            }
+        }
+    })
+}

crates/dq/src/bin/dq.rs

@@ -0,0 +1,103 @@
+use clap::ArgAction;
+use clap::Parser;
+use disk::eventchunker::EventChunker;
+use disk::eventchunker::EventChunkerConf;
+use err::Error;
+#[allow(unused)]
+use netpod::log::*;
+use netpod::range::evrange::NanoRange;
+use netpod::ByteOrder;
+use netpod::ByteSize;
+use netpod::SfChFetchInfo;
+use std::path::PathBuf;
+use tokio::fs::File;
+use tokio::io::AsyncReadExt;
+
+#[derive(Debug, Parser)]
+#[command(name = "DAQ buffer tools", author, version)]
+pub struct Opts {
+    #[arg(short, long, action(ArgAction::Count))]
+    pub verbose: u32,
+    #[command(subcommand)]
+    pub subcmd: SubCmd,
+}
+
+#[derive(Debug, Parser)]
+pub enum SubCmd {
+    ReadDatabufferConfigfile(ReadDatabufferConfigfile),
+    ReadDatabufferDatafile(ReadDatabufferDatafile),
+}
+
+#[derive(Debug, Parser)]
+pub struct ReadDatabufferConfigfile {
+    #[arg(long)]
+    configfile: PathBuf,
+}
+
+#[derive(Debug, Parser)]
+pub struct ReadDatabufferDatafile {
+    #[arg(long)]
+    configfile: PathBuf,
+    #[arg(long)]
+    datafile: PathBuf,
+}
+
+pub fn main() -> Result<(), Error> {
+    taskrun::run(async {
+        if false {
+            return Err(Error::with_msg_no_trace(format!("unknown command")));
+        }
+        let opts = Opts::parse();
+        match opts.subcmd {
+            SubCmd::ReadDatabufferConfigfile(sub) => {
+                let mut file = File::open(&sub.configfile).await?;
+                let meta = file.metadata().await?;
+                let mut buf = vec![0; meta.len() as usize];
+                file.read_exact(&mut buf).await?;
+                drop(file);
+                let config = match parse::channelconfig::parse_config(&buf) {
+                    Ok(k) => k.1,
+                    Err(e) => return Err(Error::with_msg_no_trace(format!("can not parse: {:?}", e))),
+                };
+                eprintln!("Read config: {:?}", config);
+                Ok(())
+            }
+            SubCmd::ReadDatabufferDatafile(sub) => {
+                let mut file = File::open(&sub.configfile).await?;
+                let meta = file.metadata().await?;
+                let mut buf = vec![0; meta.len() as usize];
+                file.read_exact(&mut buf).await?;
+                drop(file);
+                let config = match parse::channelconfig::parse_config(&buf) {
+                    Ok(k) => k.1,
+                    Err(e) => return Err(Error::with_msg_no_trace(format!("can not parse: {:?}", e))),
+                };
+                eprintln!("Read config: {:?}", config);
+                let path = sub.datafile;
+                let file = File::open(&path).await?;
+                let disk_io_tune = netpod::DiskIoTune::default();
+                let inp = Box::pin(disk::file_content_stream(path.clone(), file, disk_io_tune));
+                let ce = &config.entries[0];
+                let fetch_info = SfChFetchInfo::new(
+                    "",
+                    config.channel_name,
+                    ce.ks as _,
+                    ce.bs.clone(),
+                    ByteOrder::Little,
+                    ce.scalar_type.clone(),
+                    ce.to_shape()?,
+                );
+                let range = NanoRange {
+                    beg: u64::MIN,
+                    end: u64::MAX,
+                };
+                let stats_conf = EventChunkerConf {
+                    disk_stats_every: ByteSize::from_mb(2),
+                };
+                let _chunks = EventChunker::from_start(inp, fetch_info, range, stats_conf, path.clone(), false, true);
+                err::todo();
+                Ok(())
+            }
+        }
+    })
+}

crates/dq/src/dq.rs

@@ -0,0 +1,42 @@
+use bytes::BufMut;
+use std::fmt;
+
+trait WritableValue: fmt::Debug {
+    fn put_value(&self, buf: &mut Vec<u8>);
+}
+
+impl WritableValue for u32 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_u32_le(*self);
+    }
+}
+
+impl WritableValue for i8 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_i8(*self);
+    }
+}
+
+impl WritableValue for i16 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_i16_le(*self);
+    }
+}
+
+impl WritableValue for i32 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_i32_le(*self);
+    }
+}
+
+impl WritableValue for f32 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_f32_le(*self);
+    }
+}
+
+impl WritableValue for f64 {
+    fn put_value(&self, buf: &mut Vec<u8>) {
+        buf.put_f64_le(*self);
+    }
+}