daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve search api

Browse Source
This commit is contained in:
Dominik Werder
2021-11-03 16:00:53 +01:00
parent 4d3965660c
commit 96e0473392
21 changed files with 1285 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
archapp/src/archeng/datablock.rs
View File

@@ -1,9 +1,12 @@
use super::format_hex_block;
use super::indextree::DataheaderPos;
use crate::archeng::ringbuf::RingBuf;
use crate::archeng::{read_exact, read_string, readf64, readu16, readu32, seek, StatsChannel, EPICS_EPOCH_OFFSET};
use crate::eventsitem::EventsItem;
use crate::plainevents::{PlainEvents, ScalarPlainEvents};
use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use err::Error;
use items::eventvalues::EventValues;
use items::waveevents::WaveEvents;
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{NanoRange, Nanos};
@@ -11,13 +14,17 @@ use std::convert::TryInto;
use std::io::SeekFrom;
use tokio::fs::File;
use super::indextree::DataheaderPos;
#[derive(Debug)]
#[derive(Clone, Debug)]
enum DbrType {
DbrString = 0,
DbrInt = 1,
DbrShort = 1,
DbrStsFloat = 9,
DbrTimeString = 14,
DbrTimeShort = 15,
DbrTimeFloat = 16,
DbrTimeEnum = 17,
DbrTimeChar = 18,
DbrTimeLong = 19,
DbrTimeDouble = 20,
}
@@ -26,8 +33,14 @@ impl DbrType {
use DbrType::*;
let res = match k {
0 => DbrString,
1 => DbrInt,
1 => DbrShort,
9 => DbrStsFloat,
14 => DbrTimeString,
15 => DbrTimeShort,
16 => DbrTimeFloat,
17 => DbrTimeEnum,
18 => DbrTimeChar,
19 => DbrTimeLong,
20 => DbrTimeDouble,
_ => {
let msg = format!("not a valid/supported dbr type: {}", k);
@@ -37,16 +50,60 @@ impl DbrType {
Ok(res)
}
#[allow(dead_code)]
fn byte_len(&self) -> usize {
fn meta_len(&self) -> usize {
use DbrType::*;
match self {
DbrString => 0,
DbrInt => 4,
DbrStsFloat => 1,
DbrTimeDouble => 16,
DbrShort => 0,
DbrStsFloat => 4,
DbrTimeString => 12,
DbrTimeShort => 12,
DbrTimeFloat => 12,
DbrTimeEnum => 12,
DbrTimeChar => 12,
DbrTimeLong => 12,
DbrTimeDouble => 12,
}
}
fn pad_meta(&self) -> usize {
use DbrType::*;
match self {
DbrString => 0,
DbrShort => 0,
DbrStsFloat => 0,
DbrTimeString => 0,
DbrTimeShort => 2,
DbrTimeFloat => 0,
DbrTimeEnum => 2,
DbrTimeChar => 3,
DbrTimeLong => 0,
DbrTimeDouble => 4,
}
}
fn val_len(&self) -> usize {
use DbrType::*;
match self {
DbrString => 40,
DbrShort => 2,
DbrStsFloat => 4,
DbrTimeString => 40,
DbrTimeShort => 2,
DbrTimeFloat => 4,
DbrTimeEnum => 2,
DbrTimeChar => 1,
DbrTimeLong => 4,
DbrTimeDouble => 8,
}
}
fn msg_len(&self, count: usize) -> usize {
let n = self.meta_len() + self.pad_meta() + count * self.val_len();
let r = n % 8;
let n = if r == 0 { n } else { n + 8 - r };
n
}
}
#[derive(Debug)]
@@ -204,6 +261,148 @@ pub async fn read_datafile_header2(rb: &mut RingBuf<File>, pos: DataheaderPos) -
Ok(ret)
}
trait MetaParse {
fn parse_meta(buf: &[u8]) -> (u64, usize);
}
struct NoneMetaParse;
impl MetaParse for NoneMetaParse {
#[inline(always)]
fn parse_meta(_buf: &[u8]) -> (u64, usize) {
(0, 0)
}
}
struct TimeMetaParse;
impl MetaParse for TimeMetaParse {
#[inline(always)]
fn parse_meta(buf: &[u8]) -> (u64, usize) {
let tsa = u32::from_be_bytes(buf[4..8].try_into().unwrap());
let tsb = u32::from_be_bytes(buf[8..12].try_into().unwrap());
let ts = tsa as u64 * SEC + tsb as u64 + EPICS_EPOCH_OFFSET;
(ts, 12)
}
}
#[inline(always)]
fn parse_msg<MP: MetaParse, F, VT>(
buf: &[u8],
_meta_parse: MP,
dbrt: DbrType,
dbrcount: usize,
valf: F,
) -> Result<(u64, VT, usize), Error>
where
F: Fn(&[u8], usize) -> VT,
{
let (ts, n) = MP::parse_meta(buf);
let buf = &buf[n + dbrt.pad_meta()..];
Ok((ts, valf(buf, dbrcount), n))
}
macro_rules! ex_s {
($sty:ident, $n:ident) => {
fn $n(buf: &[u8], _dbrcount: usize) -> $sty {
const R: usize = std::mem::size_of::<$sty>();
$sty::from_be_bytes(buf[0..R].try_into().unwrap())
}
};
}
macro_rules! ex_v {
($sty:ident, $n:ident) => {
fn $n(mut buf: &[u8], dbrcount: usize) -> Vec<$sty> {
const R: usize = std::mem::size_of::<$sty>();
let mut a = Vec::with_capacity(dbrcount);
for _ in 0..dbrcount {
let v = $sty::from_be_bytes(buf[0..R].try_into().unwrap());
a.push(v);
buf = &buf[R..];
}
a
}
};
}
ex_s!(i8, ex_s_i8);
ex_s!(i16, ex_s_i16);
ex_s!(i32, ex_s_i32);
ex_s!(f32, ex_s_f32);
ex_s!(f64, ex_s_f64);
ex_v!(i8, ex_v_i8);
ex_v!(i16, ex_v_i16);
ex_v!(i32, ex_v_i32);
ex_v!(f32, ex_v_f32);
ex_v!(f64, ex_v_f64);
macro_rules! read_msg {
($sty:ident, $exfs:ident, $exfv:ident, $evvar:ident, $rb:expr, $msglen:expr, $numsamples:expr, $dbrt:expr, $dbrcount:ident) => {
if $dbrcount == 1 {
let mut evs = EventValues::empty();
for _ in 0..$numsamples {
$rb.fill_min($msglen).await?;
let buf = $rb.data();
let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfs)?;
evs.tss.push(ts);
evs.values.push(val);
$rb.adv($msglen);
}
let evs = ScalarPlainEvents::$evvar(evs);
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
let mut evs = WaveEvents::empty();
for _ in 0..$numsamples {
$rb.fill_min($msglen).await?;
let buf = $rb.data();
let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfv)?;
evs.tss.push(ts);
evs.vals.push(val);
$rb.adv($msglen);
}
let evs = WavePlainEvents::$evvar(evs);
let plain = PlainEvents::Wave(evs);
let item = EventsItem::Plain(plain);
item
}
};
}
async fn _format_debug_1(rb: &mut RingBuf<File>, dbrcount: usize) -> Result<(), Error> {
rb.fill_min(1024 * 10).await?;
for i1 in 0..19 {
let hex = format_hex_block(&rb.data()[512 * i1..], 512);
error!("dbrcount {} block\n{}", dbrcount, hex);
}
return Err(Error::with_msg_no_trace("EXIT"));
}
fn _format_debug_2(evs: WaveEvents<i32>) -> Result<(), Error> {
info!("tss: {:?}", evs.tss);
let n = evs.vals.len();
let vals: Vec<_> = evs
.vals
.iter()
.enumerate()
.filter(|&(i, _)| i < 3 || i + 3 >= n)
.map(|(_i, j)| {
if j.len() > 6 {
let mut a = j[0..3].to_vec();
a.extend_from_slice(&j[j.len() - 3..]);
a.to_vec()
} else {
j.to_vec()
}
})
.collect();
info!("vals: {:?}", vals);
Ok(())
}
pub async fn read_data2(
rb: &mut RingBuf<File>,
datafile_header: &DatafileHeader,
@@ -211,58 +410,55 @@ pub async fn read_data2(
_expand: bool,
) -> Result<EventsItem, Error> {
// TODO handle expand mode
//let dhpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64;
//seek(file, SeekFrom::Start(dhpos), stats).await?;
let res = match &datafile_header.dbr_type {
DbrType::DbrTimeDouble => {
if datafile_header.dbr_count == 1 {
trace!("~~~~~~~~~~~~~~~~~~~~~ read scalar DbrTimeDouble");
let mut evs = EventValues {
tss: vec![],
values: vec![],
};
let n1 = datafile_header.num_samples as usize;
//let n2 = datafile_header.dbr_type.byte_len();
let n2 = 2 + 2 + 4 + 4 + (4) + 8;
let n3 = n1 * n2;
rb.fill_min(n3).await?;
//let mut buf = vec![0; n3];
//read_exact(file, &mut buf, stats).await?;
let buf = rb.data();
let mut p1 = 0;
let mut ntot = 0;
while p1 < n3 - n2 {
let _status = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap());
p1 += 2;
let _severity = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap());
p1 += 2;
let ts1a = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap());
p1 += 4;
let ts1b = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap());
p1 += 4;
let ts1 = ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET;
p1 += 4;
let value = f64::from_be_bytes(buf[p1..p1 + 8].try_into().unwrap());
p1 += 8;
ntot += 1;
if ts1 >= range.beg && ts1 < range.end {
evs.tss.push(ts1);
evs.values.push(value);
}
}
rb.adv(n3);
//info!("parsed block with {} / {} events", ntot, evs.tss.len());
let evs = ScalarPlainEvents::Double(evs);
{
let dpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64;
if rb.rp_abs() != dpos {
warn!("read_data2 rb not positioned {} vs {}", rb.rp_abs(), dpos);
rb.seek(dpos).await?;
}
}
let numsamples = datafile_header.num_samples as usize;
let dbrcount = datafile_header.dbr_count;
let dbrt = datafile_header.dbr_type.clone();
let dbrt = if let DbrType::DbrTimeEnum = dbrt {
DbrType::DbrTimeShort
} else {
dbrt
};
let msg_len = dbrt.msg_len(dbrcount);
{
if (datafile_header.buf_size as usize) < numsamples * msg_len {
return Err(Error::with_msg_no_trace(format!(
"buffer too small for data {} {} {}",
datafile_header.buf_size, numsamples, msg_len
)));
}
}
if dbrcount == 0 {
return Err(Error::with_msg_no_trace(format!("unexpected dbrcount {}", dbrcount)));
}
let res = match &dbrt {
DbrType::DbrTimeChar => read_msg!(i8, ex_s_i8, ex_v_i8, Byte, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeShort => read_msg!(i16, ex_s_i16, ex_v_i16, Short, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeLong => read_msg!(i32, ex_s_i32, ex_v_i32, Int, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeFloat => read_msg!(f32, ex_s_f32, ex_v_f32, Float, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeDouble => read_msg!(f64, ex_s_f64, ex_v_f64, Double, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeString => {
if dbrcount == 1 {
// TODO
let evs = ScalarPlainEvents::Byte(EventValues::empty());
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
let msg = format!("dbr_count {:?} not yet supported", datafile_header.dbr_count);
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));
// TODO
let evs = WavePlainEvents::Double(WaveEvents::empty());
let plain = PlainEvents::Wave(evs);
let item = EventsItem::Plain(plain);
item
}
}
_ => {
DbrType::DbrTimeEnum | DbrType::DbrShort | DbrType::DbrString | DbrType::DbrStsFloat => {
let msg = format!("Type {:?} not yet supported", datafile_header.dbr_type);
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));