use super::format_hex_block; use super::indextree::DataheaderPos; use crate::archeng::ringbuf::RingBuf; use crate::archeng::{read_exact, read_string, readf64, readu16, readu32, seek, StatsChannel, EPICS_EPOCH_OFFSET}; use crate::eventsitem::EventsItem; use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents}; use err::Error; use items::eventvalues::EventValues; use items::waveevents::WaveEvents; use netpod::log::*; use netpod::timeunits::SEC; use netpod::{NanoRange, Nanos}; use std::convert::TryInto; use std::io::SeekFrom; use tokio::fs::File; #[derive(Clone, Debug)] enum DbrType { DbrString = 0, DbrShort = 1, DbrStsFloat = 9, DbrTimeString = 14, DbrTimeShort = 15, DbrTimeFloat = 16, DbrTimeEnum = 17, DbrTimeChar = 18, DbrTimeLong = 19, DbrTimeDouble = 20, } impl DbrType { fn from_u16(k: u16) -> Result { use DbrType::*; let res = match k { 0 => DbrString, 1 => DbrShort, 9 => DbrStsFloat, 14 => DbrTimeString, 15 => DbrTimeShort, 16 => DbrTimeFloat, 17 => DbrTimeEnum, 18 => DbrTimeChar, 19 => DbrTimeLong, 20 => DbrTimeDouble, _ => { let msg = format!("not a valid/supported dbr type: {}", k); return Err(Error::with_msg_no_trace(msg)); } }; Ok(res) } fn meta_len(&self) -> usize { use DbrType::*; match self { DbrString => 0, DbrShort => 0, DbrStsFloat => 4, DbrTimeString => 12, DbrTimeShort => 12, DbrTimeFloat => 12, DbrTimeEnum => 12, DbrTimeChar => 12, DbrTimeLong => 12, DbrTimeDouble => 12, } } fn pad_meta(&self) -> usize { use DbrType::*; match self { DbrString => 0, DbrShort => 0, DbrStsFloat => 0, DbrTimeString => 0, DbrTimeShort => 2, DbrTimeFloat => 0, DbrTimeEnum => 2, DbrTimeChar => 3, DbrTimeLong => 0, DbrTimeDouble => 4, } } fn val_len(&self) -> usize { use DbrType::*; match self { DbrString => 40, DbrShort => 2, DbrStsFloat => 4, DbrTimeString => 40, DbrTimeShort => 2, DbrTimeFloat => 4, DbrTimeEnum => 2, DbrTimeChar => 1, DbrTimeLong => 4, DbrTimeDouble => 8, } } fn msg_len(&self, count: usize) -> usize { let n = self.meta_len() + self.pad_meta() + count * self.val_len(); let r = n % 8; let n = if r == 0 { n } else { n + 8 - r }; n } } #[derive(Debug)] pub struct DatafileHeader { pos: DataheaderPos, dir_offset: u32, // Should be absolute file position of the next data header // together with `fname_next`. // But unfortunately not always set? next_offset: u32, prev_offset: u32, curr_offset: u32, pub num_samples: u32, ctrl_info_offset: u32, buf_size: u32, buf_free: u32, dbr_type: DbrType, dbr_count: usize, period: f64, ts_beg: Nanos, ts_end: Nanos, ts_next_file: Nanos, fname_next: String, fname_prev: String, } const DATA_HEADER_LEN_ON_DISK: usize = 72 + 40 + 40; // TODO retire this version (better version reads from buffer) pub async fn read_datafile_header( file: &mut File, pos: DataheaderPos, stats: &StatsChannel, ) -> Result { let mut rb = RingBuf::new(file, pos.0, stats.clone()).await?; rb.fill_min(DATA_HEADER_LEN_ON_DISK).await?; let buf = rb.data(); let dir_offset = readu32(buf, 0); let next_offset = readu32(buf, 4); let prev_offset = readu32(buf, 8); let curr_offset = readu32(buf, 12); let num_samples = readu32(buf, 16); let ctrl_info_offset = readu32(buf, 20); let buf_size = readu32(buf, 24); let buf_free = readu32(buf, 28); let dbr_type = DbrType::from_u16(readu16(buf, 32))?; let dbr_count = readu16(buf, 34); // 4 bytes padding. let period = readf64(buf, 40); let ts1a = readu32(buf, 48); let ts1b = readu32(buf, 52); let ts2a = readu32(buf, 56); let ts2b = readu32(buf, 60); let ts3a = readu32(buf, 64); let ts3b = readu32(buf, 68); let ts_beg = if ts1a != 0 || ts1b != 0 { ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let ts_end = if ts3a != 0 || ts3b != 0 { ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let ts_next_file = if ts2a != 0 || ts2b != 0 { ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let fname_prev = read_string(&buf[72..112])?; let fname_next = read_string(&buf[112..152])?; let ret = DatafileHeader { pos, dir_offset, next_offset, prev_offset, curr_offset, num_samples, ctrl_info_offset, buf_size, buf_free, dbr_type, dbr_count: dbr_count as usize, period, ts_beg: Nanos { ns: ts_beg }, ts_end: Nanos { ns: ts_end }, ts_next_file: Nanos { ns: ts_next_file }, fname_next, fname_prev, }; Ok(ret) } pub async fn read_datafile_header2(rb: &mut RingBuf, pos: DataheaderPos) -> Result { // TODO avoid the extra seek: make sure that RingBuf catches this. Profile.. rb.seek(pos.0).await?; rb.fill_min(DATA_HEADER_LEN_ON_DISK).await?; let buf = rb.data(); let dir_offset = readu32(buf, 0); let next_offset = readu32(buf, 4); let prev_offset = readu32(buf, 8); let curr_offset = readu32(buf, 12); let num_samples = readu32(buf, 16); let ctrl_info_offset = readu32(buf, 20); let buf_size = readu32(buf, 24); let buf_free = readu32(buf, 28); let dbr_type = DbrType::from_u16(readu16(buf, 32))?; let dbr_count = readu16(buf, 34); // 4 bytes padding. let period = readf64(buf, 40); let ts1a = readu32(buf, 48); let ts1b = readu32(buf, 52); let ts2a = readu32(buf, 56); let ts2b = readu32(buf, 60); let ts3a = readu32(buf, 64); let ts3b = readu32(buf, 68); let ts_beg = if ts1a != 0 || ts1b != 0 { ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let ts_end = if ts3a != 0 || ts3b != 0 { ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let ts_next_file = if ts2a != 0 || ts2b != 0 { ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET } else { 0 }; let fname_prev = read_string(&buf[72..112])?; let fname_next = read_string(&buf[112..152])?; rb.adv(DATA_HEADER_LEN_ON_DISK); let ret = DatafileHeader { pos, dir_offset, next_offset, prev_offset, curr_offset, num_samples, ctrl_info_offset, buf_size, buf_free, dbr_type, dbr_count: dbr_count as usize, period, ts_beg: Nanos { ns: ts_beg }, ts_end: Nanos { ns: ts_end }, ts_next_file: Nanos { ns: ts_next_file }, fname_next, fname_prev, }; Ok(ret) } trait MetaParse { fn parse_meta(buf: &[u8]) -> (u64, usize); } struct NoneMetaParse; impl MetaParse for NoneMetaParse { #[inline(always)] fn parse_meta(_buf: &[u8]) -> (u64, usize) { (0, 0) } } struct TimeMetaParse; impl MetaParse for TimeMetaParse { #[inline(always)] fn parse_meta(buf: &[u8]) -> (u64, usize) { let tsa = u32::from_be_bytes(buf[4..8].try_into().unwrap()); let tsb = u32::from_be_bytes(buf[8..12].try_into().unwrap()); let ts = tsa as u64 * SEC + tsb as u64 + EPICS_EPOCH_OFFSET; (ts, 12) } } #[inline(always)] fn parse_msg( buf: &[u8], _meta_parse: MP, dbrt: DbrType, dbrcount: usize, valf: F, ) -> Result<(u64, VT, usize), Error> where F: Fn(&[u8], usize) -> VT, { let (ts, n) = MP::parse_meta(buf); let buf = &buf[n + dbrt.pad_meta()..]; Ok((ts, valf(buf, dbrcount), n)) } macro_rules! ex_s { ($sty:ident, $n:ident) => { fn $n(buf: &[u8], _dbrcount: usize) -> $sty { const R: usize = std::mem::size_of::<$sty>(); $sty::from_be_bytes(buf[0..R].try_into().unwrap()) } }; } macro_rules! ex_v { ($sty:ident, $n:ident) => { fn $n(mut buf: &[u8], dbrcount: usize) -> Vec<$sty> { const R: usize = std::mem::size_of::<$sty>(); let mut a = Vec::with_capacity(dbrcount); for _ in 0..dbrcount { let v = $sty::from_be_bytes(buf[0..R].try_into().unwrap()); a.push(v); buf = &buf[R..]; } a } }; } ex_s!(i8, ex_s_i8); ex_s!(i16, ex_s_i16); ex_s!(i32, ex_s_i32); ex_s!(f32, ex_s_f32); ex_s!(f64, ex_s_f64); ex_v!(i8, ex_v_i8); ex_v!(i16, ex_v_i16); ex_v!(i32, ex_v_i32); ex_v!(f32, ex_v_f32); ex_v!(f64, ex_v_f64); macro_rules! read_msg { ($sty:ident, $exfs:ident, $exfv:ident, $evvar:ident, $rb:expr, $msglen:expr, $numsamples:expr, $dbrt:expr, $dbrcount:ident) => { if $dbrcount == 1 { let mut evs = EventValues::empty(); for _ in 0..$numsamples { $rb.fill_min($msglen).await?; let buf = $rb.data(); let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfs)?; evs.tss.push(ts); evs.values.push(val); $rb.adv($msglen); } let evs = ScalarPlainEvents::$evvar(evs); let plain = PlainEvents::Scalar(evs); let item = EventsItem::Plain(plain); item } else { let mut evs = WaveEvents::empty(); for _ in 0..$numsamples { $rb.fill_min($msglen).await?; let buf = $rb.data(); let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfv)?; evs.tss.push(ts); evs.vals.push(val); $rb.adv($msglen); } let evs = WavePlainEvents::$evvar(evs); let plain = PlainEvents::Wave(evs); let item = EventsItem::Plain(plain); item } }; } async fn _format_debug_1(rb: &mut RingBuf, dbrcount: usize) -> Result<(), Error> { rb.fill_min(1024 * 10).await?; for i1 in 0..19 { let hex = format_hex_block(&rb.data()[512 * i1..], 512); error!("dbrcount {} block\n{}", dbrcount, hex); } return Err(Error::with_msg_no_trace("EXIT")); } fn _format_debug_2(evs: WaveEvents) -> Result<(), Error> { info!("tss: {:?}", evs.tss); let n = evs.vals.len(); let vals: Vec<_> = evs .vals .iter() .enumerate() .filter(|&(i, _)| i < 3 || i + 3 >= n) .map(|(_i, j)| { if j.len() > 6 { let mut a = j[0..3].to_vec(); a.extend_from_slice(&j[j.len() - 3..]); a.to_vec() } else { j.to_vec() } }) .collect(); info!("vals: {:?}", vals); Ok(()) } pub async fn read_data2( rb: &mut RingBuf, datafile_header: &DatafileHeader, range: NanoRange, _expand: bool, ) -> Result { // TODO handle expand mode { let dpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64; if rb.rp_abs() != dpos { warn!("read_data2 rb not positioned {} vs {}", rb.rp_abs(), dpos); rb.seek(dpos).await?; } } let numsamples = datafile_header.num_samples as usize; let dbrcount = datafile_header.dbr_count; let dbrt = datafile_header.dbr_type.clone(); let dbrt = if let DbrType::DbrTimeEnum = dbrt { DbrType::DbrTimeShort } else { dbrt }; let msg_len = dbrt.msg_len(dbrcount); { if (datafile_header.buf_size as usize) < numsamples * msg_len { return Err(Error::with_msg_no_trace(format!( "buffer too small for data {} {} {}", datafile_header.buf_size, numsamples, msg_len ))); } } if dbrcount == 0 { return Err(Error::with_msg_no_trace(format!("unexpected dbrcount {}", dbrcount))); } let res = match &dbrt { DbrType::DbrTimeChar => read_msg!(i8, ex_s_i8, ex_v_i8, Byte, rb, msg_len, numsamples, dbrt, dbrcount), DbrType::DbrTimeShort => read_msg!(i16, ex_s_i16, ex_v_i16, Short, rb, msg_len, numsamples, dbrt, dbrcount), DbrType::DbrTimeLong => read_msg!(i32, ex_s_i32, ex_v_i32, Int, rb, msg_len, numsamples, dbrt, dbrcount), DbrType::DbrTimeFloat => read_msg!(f32, ex_s_f32, ex_v_f32, Float, rb, msg_len, numsamples, dbrt, dbrcount), DbrType::DbrTimeDouble => read_msg!(f64, ex_s_f64, ex_v_f64, Double, rb, msg_len, numsamples, dbrt, dbrcount), DbrType::DbrTimeString => { if dbrcount == 1 { // TODO let evs = ScalarPlainEvents::Byte(EventValues::empty()); let plain = PlainEvents::Scalar(evs); let item = EventsItem::Plain(plain); item } else { // TODO let evs = WavePlainEvents::Double(WaveEvents::empty()); let plain = PlainEvents::Wave(evs); let item = EventsItem::Plain(plain); item } } DbrType::DbrTimeEnum | DbrType::DbrShort | DbrType::DbrString | DbrType::DbrStsFloat => { let msg = format!("Type {:?} not yet supported", datafile_header.dbr_type); error!("{}", msg); return Err(Error::with_msg_no_trace(msg)); } }; Ok(res) } pub async fn read_data_1( file: &mut File, datafile_header: &DatafileHeader, range: NanoRange, _expand: bool, stats: &StatsChannel, ) -> Result { // TODO handle expand mode let dhpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64; seek(file, SeekFrom::Start(dhpos), stats).await?; let res = match &datafile_header.dbr_type { DbrType::DbrTimeDouble => { if datafile_header.dbr_count == 1 { trace!("~~~~~~~~~~~~~~~~~~~~~ read scalar DbrTimeDouble"); let mut evs = EventValues { tss: vec![], values: vec![], }; let n1 = datafile_header.num_samples as usize; //let n2 = datafile_header.dbr_type.byte_len(); let n2 = 2 + 2 + 4 + 4 + (4) + 8; let n3 = n1 * n2; let mut buf = vec![0; n3]; read_exact(file, &mut buf, stats).await?; let mut p1 = 0; let mut ntot = 0; while p1 < n3 - n2 { let _status = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap()); p1 += 2; let _severity = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap()); p1 += 2; let ts1a = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap()); p1 += 4; let ts1b = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap()); p1 += 4; let ts1 = ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET; p1 += 4; let value = f64::from_be_bytes(buf[p1..p1 + 8].try_into().unwrap()); p1 += 8; ntot += 1; if ts1 >= range.beg && ts1 < range.end { evs.tss.push(ts1); evs.values.push(value); } } info!("parsed block with {} / {} events", ntot, evs.tss.len()); let evs = ScalarPlainEvents::Double(evs); let plain = PlainEvents::Scalar(evs); let item = EventsItem::Plain(plain); item } else { let msg = format!("dbr_count {:?} not yet supported", datafile_header.dbr_count); error!("{}", msg); return Err(Error::with_msg_no_trace(msg)); } } _ => { let msg = format!("Type {:?} not yet supported", datafile_header.dbr_type); error!("{}", msg); return Err(Error::with_msg_no_trace(msg)); } }; Ok(res) }