diff --git a/archapp/Cargo.toml b/archapp/Cargo.toml index 596d8ba..bb66cf3 100644 --- a/archapp/Cargo.toml +++ b/archapp/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Dominik Werder "] edition = "2018" [dependencies] -tokio = { version = "1.7.1", features = ["io-util", "net", "time", "sync", "fs"] } +tokio = { version = "1.7.1", features = ["io-util", "net", "time", "sync", "fs", "parking_lot"] } tracing = "0.1.26" futures-core = "0.3.15" futures-util = "0.3.15" @@ -17,6 +17,10 @@ bincode = "1.3.3" chrono = "0.4.19" protobuf = "2.24.1" async-channel = "1.6" +parking_lot = "0.11.2" +crc32fast = "1.2.1" +regex = "1.5.4" +tokio-postgres = { version = "0.7.4", features = ["runtime", "with-chrono-0_4", "with-serde_json-1"] } archapp_xc = { path = "../archapp_xc" } err = { path = "../err" } taskrun = { path = "../taskrun" } diff --git a/archapp/src/archeng.rs b/archapp/src/archeng.rs index 81bc567..d3b5e3e 100644 --- a/archapp/src/archeng.rs +++ b/archapp/src/archeng.rs @@ -1,23 +1,33 @@ pub mod datablockstream; -pub mod datastream; +pub mod indexfiles; pub mod pipe; -use crate::{EventsItem, PlainEvents, ScalarPlainEvents}; +use self::indexfiles::list_index_files; +use crate::eventsitem::EventsItem; +use crate::plainevents::{PlainEvents, ScalarPlainEvents}; +use crate::timed::Timed; +use crate::wrap_task; use async_channel::{Receiver, Sender}; use err::Error; -use futures_core::Future; use futures_util::StreamExt; use items::eventvalues::EventValues; -use items::{RangeCompletableItem, StreamItem}; +use items::{RangeCompletableItem, Sitemty, StatsItem, StreamItem}; use netpod::timeunits::SEC; -use netpod::{log::*, ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse, DataHeaderPos, FilePos, Nanos}; +use netpod::{ + log::*, Channel, ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse, DataHeaderPos, DiskStats, FilePos, + NanoRange, Nanos, OpenStats, ReadExactStats, ReadStats, SeekStats, +}; +use regex::Regex; use serde::Serialize; +use std::collections::BTreeMap; use std::convert::TryInto; use std::io::{self, SeekFrom}; use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; -use tokio::fs::{read_dir, File, OpenOptions}; +use tokio::fs::{File, OpenOptions}; use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio::sync::Mutex; /* struct ReadExactWrap<'a> { @@ -45,47 +55,130 @@ type Offset = u64; const OFFSET_SIZE: usize = std::mem::size_of::(); const EPICS_EPOCH_OFFSET: u64 = 631152000 * SEC; +const LOG_IO: bool = true; +const STATS_IO: bool = true; +static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0); -pub async fn open_read(path: PathBuf) -> io::Result { +fn channel_send_error() { + let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel); + if c < 10 { + error!("CHANNEL_SEND_ERROR {}", c); + } +} + +pub struct StatsChannel { + chn: Sender>, +} + +impl StatsChannel { + pub fn new(chn: Sender>) -> Self { + Self { chn } + } + + pub fn dummy() -> Self { + let (tx, rx) = async_channel::bounded(2); + taskrun::spawn(async move { + let mut rx = rx; + while let Some(_) = rx.next().await {} + }); + Self::new(tx) + } + + pub async fn send(&self, item: StatsItem) -> Result<(), Error> { + Ok(self.chn.send(Ok(StreamItem::Stats(item))).await?) + } +} + +impl Clone for StatsChannel { + fn clone(&self) -> Self { + Self { chn: self.chn.clone() } + } +} + +pub async fn open_read(path: PathBuf, stats: &StatsChannel) -> io::Result { let ts1 = Instant::now(); let res = OpenOptions::new().read(true).open(path).await; let ts2 = Instant::now(); - let dt = ts2.duration_since(ts1).as_secs_f64() * 1e3; - if false { - info!("timed open_read dt: {:.3} ms", dt); + let dt = ts2.duration_since(ts1); + if LOG_IO { + let dt = dt.as_secs_f64() * 1e3; + debug!("timed open_read dt: {:.3} ms", dt); + } + if STATS_IO { + if let Err(_) = stats + .send(StatsItem::DiskStats(DiskStats::OpenStats(OpenStats::new( + ts2.duration_since(ts1), + )))) + .await + { + channel_send_error(); + } } res } -async fn seek(file: &mut File, pos: SeekFrom) -> io::Result { +async fn seek(file: &mut File, pos: SeekFrom, stats: &StatsChannel) -> io::Result { let ts1 = Instant::now(); let res = file.seek(pos).await; let ts2 = Instant::now(); - let dt = ts2.duration_since(ts1).as_secs_f64() * 1e3; - if false { - info!("timed seek dt: {:.3} ms", dt); + let dt = ts2.duration_since(ts1); + if LOG_IO { + let dt = dt.as_secs_f64() * 1e3; + debug!("timed seek dt: {:.3} ms", dt); + } + if STATS_IO { + if let Err(_) = stats + .send(StatsItem::DiskStats(DiskStats::SeekStats(SeekStats::new( + ts2.duration_since(ts1), + )))) + .await + { + channel_send_error(); + } } res } -async fn read(file: &mut File, buf: &mut [u8]) -> io::Result { +async fn read(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result { let ts1 = Instant::now(); let res = file.read(buf).await; let ts2 = Instant::now(); - let dt = ts2.duration_since(ts1).as_secs_f64() * 1e3; - if false { - info!("timed read dt: {:.3} ms res: {:?}", dt, res); + let dt = ts2.duration_since(ts1); + if LOG_IO { + let dt = dt.as_secs_f64() * 1e3; + debug!("timed read dt: {:.3} ms res: {:?}", dt, res); + } + if STATS_IO { + if let Err(_) = stats + .send(StatsItem::DiskStats(DiskStats::ReadStats(ReadStats::new( + ts2.duration_since(ts1), + )))) + .await + { + channel_send_error(); + } } res } -async fn read_exact(file: &mut File, buf: &mut [u8]) -> io::Result { +async fn read_exact(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result { let ts1 = Instant::now(); let res = file.read_exact(buf).await; let ts2 = Instant::now(); - let dt = ts2.duration_since(ts1).as_secs_f64() * 1e3; - if false { - info!("timed read_exact dt: {:.3} ms res: {:?}", dt, res); + let dt = ts2.duration_since(ts1); + if LOG_IO { + let dt = dt.as_secs_f64() * 1e3; + debug!("timed read_exact dt: {:.3} ms res: {:?}", dt, res); + } + if STATS_IO { + if let Err(_) = stats + .send(StatsItem::DiskStats(DiskStats::ReadExactStats(ReadExactStats::new( + ts2.duration_since(ts1), + )))) + .await + { + channel_send_error(); + }; } res } @@ -160,7 +253,7 @@ impl RingBuf { &self.buf[self.rp..self.wp] } - pub async fn fill(&mut self, file: &mut File) -> Result { + pub async fn fill(&mut self, file: &mut File, stats: &StatsChannel) -> Result { if self.rp == self.wp { if self.rp != 0 { self.wp = 0; @@ -173,16 +266,16 @@ impl RingBuf { self.rp = 0; } } - let n = read(file, &mut self.buf[self.wp..]).await?; + let n = read(file, &mut self.buf[self.wp..], stats).await?; self.wp += n; return Ok(n); } - pub async fn fill_if_low(&mut self, file: &mut File) -> Result { + pub async fn fill_if_low(&mut self, file: &mut File, stats: &StatsChannel) -> Result { let len = self.len(); let cap = self.buf.len(); while self.len() < cap / 6 { - let n = self.fill(file).await?; + let n = self.fill(file, stats).await?; if n == 0 { break; } @@ -190,10 +283,10 @@ impl RingBuf { return Ok(self.len() - len); } - pub async fn fill_min(&mut self, file: &mut File, min: usize) -> Result { + pub async fn fill_min(&mut self, file: &mut File, min: usize, stats: &StatsChannel) -> Result { let len = self.len(); while self.len() < min { - let n = self.fill(file).await?; + let n = self.fill(file, stats).await?; if n == 0 { break; } @@ -253,14 +346,14 @@ fn read_string(buf: &[u8]) -> Result { Ok(ret) } -pub async fn read_file_basics(file: &mut File) -> Result { +pub async fn read_file_basics(file: &mut File, stats: &StatsChannel) -> Result { let mut buf = vec![0; 128]; - read_exact(file, &mut buf[0..4]).await?; + read_exact(file, &mut buf[0..4], stats).await?; let version = String::from_utf8(buf[3..4].to_vec())?.parse()?; if version == 3 { - read_exact(file, &mut buf[4..88]).await?; + read_exact(file, &mut buf[4..88], stats).await?; } else if version == 2 { - read_exact(file, &mut buf[4..48]).await?; + read_exact(file, &mut buf[4..48], stats).await?; } else { panic!(); } @@ -339,7 +432,7 @@ pub async fn read_file_basics(file: &mut File) -> Result panic!() }; buf.resize(u as usize, 0); - read_exact(file, &mut buf).await?; + read_exact(file, &mut buf, stats).await?; let b = &buf; for i1 in 0..ret.name_hash_anchor_len { let pos = if version == 3 { @@ -387,14 +480,19 @@ impl RTreeNodeAtRecord { } // TODO refactor as struct, rtree_m is a property of the tree. -pub async fn read_rtree_node(file: &mut File, pos: FilePos, rtree_m: usize) -> Result { +pub async fn read_rtree_node( + file: &mut File, + pos: FilePos, + rtree_m: usize, + stats: &StatsChannel, +) -> Result { const OFF1: usize = 9; const RLEN: usize = 24; const NANO_MAX: u32 = 999999999; - seek(file, SeekFrom::Start(pos.into())).await?; + seek(file, SeekFrom::Start(pos.into()), stats).await?; let mut rb = RingBuf::new(); // TODO must know how much data I need at least... - rb.fill_min(file, OFF1 + rtree_m * RLEN).await?; + rb.fill_min(file, OFF1 + rtree_m * RLEN, stats).await?; if false { let s = format_hex_block(rb.data(), 128); info!("RTREE NODE:\n{}", s); @@ -422,7 +520,7 @@ pub async fn read_rtree_node(file: &mut File, pos: FilePos, rtree_m: usize) -> R let ts2 = ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET; let child_or_id = readu64(b, off2 + 16); //info!("NODE {} {} {} {} {}", ts1a, ts1b, ts2a, ts2b, child_or_id); - if child_or_id != 0 { + if child_or_id != 0 && ts2 != 0 { let rec = RTreeNodeRecord { ts1: Nanos { ns: ts1 }, ts2: Nanos { ns: ts2 }, @@ -443,12 +541,17 @@ pub async fn read_rtree_node(file: &mut File, pos: FilePos, rtree_m: usize) -> R Ok(node) } -pub async fn read_rtree_entrypoint(file: &mut File, pos: u64, _basics: &IndexFileBasics) -> Result { - seek(file, SeekFrom::Start(pos)).await?; +pub async fn read_rtree_entrypoint( + file: &mut File, + pos: u64, + _basics: &IndexFileBasics, + stats: &StatsChannel, +) -> Result { + seek(file, SeekFrom::Start(pos), stats).await?; let mut rb = RingBuf::new(); // TODO should be able to indicate: // • how much I need at most before I know that I will e.g. seek or abort. - rb.fill_min(file, OFFSET_SIZE + 4).await?; + rb.fill_min(file, OFFSET_SIZE + 4, stats).await?; if rb.len() < OFFSET_SIZE + 4 { return Err(Error::with_msg_no_trace("could not read enough")); } @@ -457,7 +560,7 @@ pub async fn read_rtree_entrypoint(file: &mut File, pos: u64, _basics: &IndexFil let rtree_m = readu32(b, OFFSET_SIZE); //info!("node_offset: {} rtree_m: {}", node_offset, rtree_m); let pos = FilePos { pos: node_offset }; - let node = read_rtree_node(file, pos, rtree_m as usize).await?; + let node = read_rtree_node(file, pos, rtree_m as usize, stats).await?; //info!("read_rtree_entrypoint READ ROOT NODE: {:?}", node); Ok(node) } @@ -482,14 +585,14 @@ pub async fn search_record( rtree_m: usize, start_node_pos: FilePos, beg: Nanos, + stats: &StatsChannel, ) -> Result<(Option, TreeSearchStats), Error> { let ts1 = Instant::now(); - let mut node = read_rtree_node(file, start_node_pos, rtree_m).await?; + let mut node = read_rtree_node(file, start_node_pos, rtree_m, stats).await?; let mut node_reads = 1; 'outer: loop { let nr = node.records.len(); for (i, rec) in node.records.iter().enumerate() { - //info!("looking at record i {}", i); if rec.ts2.ns > beg.ns { if node.is_leaf { trace!("found leaf match at {} / {}", i, nr); @@ -499,7 +602,7 @@ pub async fn search_record( } else { trace!("found non-leaf match at {} / {}", i, nr); let pos = FilePos { pos: rec.child_or_id }; - node = read_rtree_node(file, pos, rtree_m).await?; + node = read_rtree_node(file, pos, rtree_m, stats).await?; node_reads += 1; continue 'outer; } @@ -512,6 +615,68 @@ pub async fn search_record( } } +pub async fn search_record_expand_try( + file: &mut File, + rtree_m: usize, + start_node_pos: FilePos, + beg: Nanos, + stats: &StatsChannel, +) -> Result<(Option, TreeSearchStats), Error> { + let ts1 = Instant::now(); + let mut node = read_rtree_node(file, start_node_pos, rtree_m, stats).await?; + let mut node_reads = 1; + 'outer: loop { + let nr = node.records.len(); + for (i, rec) in node.records.iter().enumerate().rev() { + if rec.ts1.ns <= beg.ns { + if node.is_leaf { + trace!("found leaf match at {} / {}", i, nr); + let ret = RTreeNodeAtRecord { node, rix: i }; + let stats = TreeSearchStats::new(ts1, node_reads); + return Ok((Some(ret), stats)); + } else { + // TODO + // We rely on channel archiver engine that there is at least one event + // in the referenced range. It should according to docs, but who knows. + trace!("found non-leaf match at {} / {}", i, nr); + let pos = FilePos { pos: rec.child_or_id }; + node = read_rtree_node(file, pos, rtree_m, stats).await?; + node_reads += 1; + continue 'outer; + } + } + } + { + let stats = TreeSearchStats::new(ts1, node_reads); + return Ok((None, stats)); + } + } +} + +pub async fn search_record_expand( + file: &mut File, + rtree_m: usize, + start_node_pos: FilePos, + beg: Nanos, + stats: &StatsChannel, +) -> Result<(Option, TreeSearchStats), Error> { + let ts1 = Instant::now(); + let res = search_record_expand_try(file, rtree_m, start_node_pos, beg, stats).await?; + match res { + (Some(res), stats) => { + let ts2 = Instant::now(); + info!("search_record_expand took {:?}", ts2.duration_since(ts1)); + Ok((Some(res), stats)) + } + _ => { + let res = search_record(file, rtree_m, start_node_pos, beg, stats).await; + let ts2 = Instant::now(); + info!("search_record_expand took {:?}", ts2.duration_since(ts1)); + res + } + } +} + #[derive(Debug)] pub struct ChannelInfoBasics { channel_name: String, @@ -519,8 +684,12 @@ pub struct ChannelInfoBasics { rtree_start_pos: FilePos, } -pub async fn read_channel(index_file: &mut File, channel_name: &str) -> Result, Error> { - let basics = read_file_basics(index_file).await?; +pub async fn read_channel( + index_file: &mut File, + channel_name: &str, + stats: &StatsChannel, +) -> Result, Error> { + let basics = read_file_basics(index_file, stats).await?; let hver2 = HeaderVersion2; let hver3 = HeaderVersion3; let hver: &dyn HeaderVersion = if basics.version == 3 { @@ -537,9 +706,9 @@ pub async fn read_channel(index_file: &mut File, channel_name: &str) -> Result Result Result, Error> { let mut pos = pos; let mut ret = vec![]; let mut rb = RingBuf::new(); loop { rb.reset(); - seek(file, SeekFrom::Start(pos.pos)).await?; + seek(file, SeekFrom::Start(pos.pos), stats).await?; let fill_min = if hver.offset_size() == 8 { 20 } else { 12 }; - rb.fill_min(file, fill_min).await?; + rb.fill_min(file, fill_min, stats).await?; if rb.len() < fill_min { warn!("not enough data to continue reading channel list from name hash list"); break; @@ -668,10 +838,10 @@ async fn channel_list_from_index_name_hash_list( Ok(ret) } -pub async fn channel_list(index_path: PathBuf) -> Result, Error> { +pub async fn channel_list(index_path: PathBuf, stats: &StatsChannel) -> Result, Error> { let mut ret = vec![]; - let file = &mut open_read(index_path.clone()).await?; - let basics = read_file_basics(file).await?; + let file = &mut open_read(index_path.clone(), stats).await?; + let basics = read_file_basics(file, stats).await?; let hver2 = HeaderVersion2; let hver3 = HeaderVersion3; let hver: &dyn HeaderVersion = if basics.version == 2 { @@ -689,7 +859,7 @@ pub async fn channel_list(index_path: PathBuf) -> Result, Error> { let pos = FilePos { pos: name_hash_entry.named_hash_channel_entry_pos, }; - let list = channel_list_from_index_name_hash_list(file, pos, hver).await?; + let list = channel_list_from_index_name_hash_list(file, pos, hver, stats).await?; for e in list { ret.push(e.channel_name); } @@ -733,15 +903,15 @@ impl Datablock { } } -async fn read_index_datablockref(file: &mut File, pos: FilePos) -> Result { - seek(file, SeekFrom::Start(pos.pos)).await?; +async fn read_index_datablockref(file: &mut File, pos: FilePos, stats: &StatsChannel) -> Result { + seek(file, SeekFrom::Start(pos.pos), stats).await?; let mut rb = RingBuf::new(); - rb.fill_min(file, 18).await?; + rb.fill_min(file, 18, stats).await?; let buf = rb.data(); let next = readoffset(buf, 0); let data = readoffset(buf, 8); let len = readu16(buf, 16) as usize; - rb.fill_min(file, 18 + len).await?; + rb.fill_min(file, 18 + len, stats).await?; let buf = rb.data(); let fname = String::from_utf8(buf[18..18 + len].to_vec())?; let ret = Datablock { @@ -814,10 +984,14 @@ pub struct DatafileHeader { const DATA_HEADER_LEN_ON_DISK: usize = 72 + 40 + 40; -async fn read_datafile_header(file: &mut File, pos: DataHeaderPos) -> Result { - seek(file, SeekFrom::Start(pos.0)).await?; +async fn read_datafile_header( + file: &mut File, + pos: DataHeaderPos, + stats: &StatsChannel, +) -> Result { + seek(file, SeekFrom::Start(pos.0), stats).await?; let mut rb = RingBuf::new(); - rb.fill_min(file, DATA_HEADER_LEN_ON_DISK).await?; + rb.fill_min(file, DATA_HEADER_LEN_ON_DISK, stats).await?; let buf = rb.data(); let dir_offset = readu32(buf, 0); let next_offset = readu32(buf, 4); @@ -876,9 +1050,16 @@ async fn read_datafile_header(file: &mut File, pos: DataHeaderPos) -> Result Result { +async fn read_data_1( + file: &mut File, + datafile_header: &DatafileHeader, + range: NanoRange, + _expand: bool, + stats: &StatsChannel, +) -> Result { + // TODO handle expand mode let dhpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64; - seek(file, SeekFrom::Start(dhpos)).await?; + seek(file, SeekFrom::Start(dhpos), stats).await?; let res = match &datafile_header.dbr_type { DbrType::DbrTimeDouble => { if datafile_header.dbr_count == 1 { @@ -892,8 +1073,9 @@ async fn read_data_1(file: &mut File, datafile_header: &DatafileHeader) -> Resul let n2 = 2 + 2 + 4 + 4 + (4) + 8; let n3 = n1 * n2; let mut buf = vec![0; n3]; - read_exact(file, &mut buf).await?; + read_exact(file, &mut buf, stats).await?; let mut p1 = 0; + let mut ntot = 0; while p1 < n3 - n2 { let _status = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap()); p1 += 2; @@ -907,10 +1089,13 @@ async fn read_data_1(file: &mut File, datafile_header: &DatafileHeader) -> Resul p1 += 4; let value = f64::from_be_bytes(buf[p1..p1 + 8].try_into().unwrap()); p1 += 8; - //info!("read event {} {} {} {} {}", status, severity, ts1a, ts1b, value); - evs.tss.push(ts1); - evs.values.push(value); + ntot += 1; + if ts1 >= range.beg && ts1 < range.end { + evs.tss.push(ts1); + evs.values.push(value); + } } + info!("parsed block with {} / {} events", ntot, evs.tss.len()); let evs = ScalarPlainEvents::Double(evs); let plain = PlainEvents::Scalar(evs); let item = EventsItem::Plain(plain); @@ -930,72 +1115,6 @@ async fn read_data_1(file: &mut File, datafile_header: &DatafileHeader) -> Resul Ok(res) } -pub fn list_index_files(node: &ChannelArchiver) -> Receiver> { - let node = node.clone(); - let (tx, rx) = async_channel::bounded(4); - let tx2 = tx.clone(); - let task = async move { - for bp in &node.data_base_paths { - let mut rd = read_dir(bp).await?; - while let Some(e) = rd.next_entry().await? { - let ft = e.file_type().await?; - if ft.is_dir() { - let mut rd = read_dir(e.path()).await?; - while let Some(e) = rd.next_entry().await? { - let ft = e.file_type().await?; - if false && ft.is_dir() { - let mut rd = read_dir(e.path()).await?; - while let Some(e) = rd.next_entry().await? { - let ft = e.file_type().await?; - if ft.is_file() { - if e.file_name().to_string_lossy() == "index" { - tx.send(Ok(e.path())).await?; - } - } - } - } else if ft.is_file() { - if e.file_name().to_string_lossy() == "index" { - tx.send(Ok(e.path())).await?; - } - } - } - } else if ft.is_file() { - if e.file_name().to_string_lossy() == "index" { - tx.send(Ok(e.path())).await?; - } - } - } - } - Ok::<_, Error>(()) - }; - wrap_task(task, tx2); - rx -} - -fn wrap_task(task: T, tx: Sender>) -where - T: Future> + Send + 'static, - O1: Send + 'static, - O2: Send + 'static, -{ - let task = async move { - match task.await { - Ok(_) => {} - Err(e) => { - match tx.send(Err(e)).await { - Ok(_) => {} - Err(e) => { - if false { - error!("wrap_task can not forward error: {:?}", e); - } - } - }; - } - } - }; - taskrun::spawn(task); -} - #[derive(Debug, Serialize)] pub struct ListChannelItem { name: String, @@ -1007,12 +1126,27 @@ pub fn list_all_channels(node: &ChannelArchiver) -> Receiver { + info!("stats: {:?}", item); + } + _ => {} + } + } + }); + StatsChannel::new(tx.clone()) + }; let task = async move { let mut ixf = list_index_files(&node); while let Some(f) = ixf.next().await { let index_path = f?; //info!("try to read for {:?}", index_path); - let channels = channel_list(index_path.clone()).await?; + let channels = channel_list(index_path.clone(), &stats).await?; //info!("list_all_channels emit {} channels", channels.len()); for ch in channels { let mm = match ch.split("-").next() { @@ -1043,13 +1177,16 @@ pub fn list_all_channels(node: &ChannelArchiver) -> Receiver Result { + let _timed = Timed::new("channel_config"); let mut type_info = None; let mut stream = datablockstream::DatablockStream::for_channel_range( q.range.clone(), q.channel.clone(), conf.data_base_paths.clone().into(), true, + 1, ); + let timed_expand = Timed::new("channel_config EXPAND"); while let Some(item) = stream.next().await { match item { Ok(k) => match k { @@ -1069,12 +1206,16 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R } } } + drop(timed_expand); if type_info.is_none() { + let timed_normal = Timed::new("channel_config NORMAL"); + warn!("channel_config expand mode returned none"); let mut stream = datablockstream::DatablockStream::for_channel_range( q.range.clone(), q.channel.clone(), conf.data_base_paths.clone().into(), false, + u64::MAX, ); while let Some(item) = stream.next().await { match item { @@ -1095,6 +1236,7 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R } } } + drop(timed_normal); } if let Some(type_info) = type_info { let ret = ChannelConfigResponse { @@ -1109,6 +1251,189 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R } } +#[derive(Debug)] +enum RetClass { + Long, + Medium, + Short, + #[allow(unused)] + PostMortem, +} + +#[derive(Debug)] +enum IndexCat { + Machine { rc: RetClass }, + Beamline { rc: RetClass, name: String }, +} + +#[derive(Debug)] +struct IndexFile { + path: PathBuf, + cat: IndexCat, +} + +// Try to make sense of historical conventions how the epics channel archiver engines are configured. +fn categorize_index_files(list: &Vec) -> Result, Error> { + let re_m = Regex::new(r"/archive_(ST|MT|LT)/index").unwrap(); + let re_b = Regex::new(r"/archive_(X([0-9]+)[^_]*)_(SH|LO)/index").unwrap(); + let mut ret = vec![]; + for p in list { + match re_m.captures(p) { + Some(cap) => { + let rc = cap.get(1).unwrap().as_str(); + let rc = match rc { + "ST" => Some(RetClass::Short), + "MT" => Some(RetClass::Medium), + "LT" => Some(RetClass::Long), + _ => { + warn!("categorize_index_files no idea about RC for {}", p); + None + } + }; + if let Some(rc) = rc { + let f = IndexFile { + path: p.into(), + cat: IndexCat::Machine { rc }, + }; + ret.push(f); + } + } + None => match re_b.captures(p) { + Some(cap) => { + let name = cap.get(1).unwrap().as_str(); + let rc = cap.get(3).unwrap().as_str(); + let rc = match rc { + "SH" => Some(RetClass::Short), + "LO" => Some(RetClass::Long), + _ => { + warn!("categorize_index_files no idea about RC for {}", p); + None + } + }; + if let Some(rc) = rc { + let f = IndexFile { + path: p.into(), + cat: IndexCat::Beamline { name: name.into(), rc }, + }; + ret.push(f); + } + } + None => { + warn!("categorize_index_files no idea at all about {}", p); + } + }, + } + } + let is_machine = { + let mut k = false; + for x in &ret { + if let IndexCat::Machine { .. } = &x.cat { + k = true; + break; + } + } + k + }; + // TODO by default, filter post-mortem. + let is_beamline = !is_machine; + if is_beamline { + let mut ret: Vec<_> = ret + .into_iter() + .filter_map(|k| { + if let IndexCat::Machine { rc, .. } = &k.cat { + let prio = match rc { + &RetClass::Short => 4, + &RetClass::Medium => 6, + &RetClass::Long => 8, + &RetClass::PostMortem => 0, + }; + Some((k, prio)) + } else { + None + } + }) + .collect(); + ret.sort_by_key(|x| x.1); + let ret = ret.into_iter().map(|k| k.0).collect(); + Ok(ret) + } else if is_machine { + let mut ret: Vec<_> = ret + .into_iter() + .filter_map(|k| { + if let IndexCat::Machine { rc, .. } = &k.cat { + let prio = match rc { + &RetClass::Short => 4, + &RetClass::Medium => 6, + &RetClass::Long => 8, + &RetClass::PostMortem => 0, + }; + Some((k, prio)) + } else { + None + } + }) + .collect(); + ret.sort_by_key(|x| x.1); + let ret = ret.into_iter().map(|k| k.0).collect(); + Ok(ret) + } else { + err::todoval() + } +} + +static INDEX_JSON: Mutex>>> = Mutex::const_new(None); + +pub async fn index_files_index_ref + Send>( + key: &str, + index_files_index_path: P, + stats: &StatsChannel, +) -> Result>, Error> { + let mut g = INDEX_JSON.lock().await; + match &*g { + Some(j) => Ok(j.get(key).map(|x| x.clone())), + None => { + let timed1 = Timed::new("slurp_index_json"); + let index_files_index_path = index_files_index_path.into(); + let index_files_index = { + let timed1 = Timed::new("slurp_index_bytes"); + let mut index_files_index = open_read(index_files_index_path, stats).await?; + let mut buf = vec![0; 1024 * 1024 * 50]; + let mut ntot = 0; + loop { + let n = read(&mut index_files_index, &mut buf[ntot..], stats).await?; + if n == 0 { + break; + } + ntot += n; + } + buf.truncate(ntot); + drop(timed1); + serde_json::from_slice::>>(&buf)? + }; + drop(timed1); + let ret = index_files_index.get(key).map(|x| x.clone()); + *g = Some(index_files_index); + Ok(ret) + } + } +} + +pub async fn index_file_path_list( + channel: Channel, + index_files_index_path: PathBuf, + stats: &StatsChannel, +) -> Result, Error> { + let timed1 = Timed::new("categorize index files"); + let index_paths = index_files_index_ref(channel.name(), &index_files_index_path, stats) + .await? + .ok_or(Error::with_msg_no_trace("can not find channel"))?; + let list = categorize_index_files(&index_paths)?; + info!("GOT CATEGORIZED:\n{:?}", list); + let ret = list.into_iter().map(|k| k.path).collect(); + drop(timed1); + Ok(ret) +} + #[cfg(test)] mod test { // TODO move RangeFilter to a different crate (items?) @@ -1118,13 +1443,15 @@ mod test { //use disk::{eventblobs::EventChunkerMultifile, eventchunker::EventChunkerConf}; use super::search_record; - use crate::archeng::{open_read, read_channel, read_data_1, read_file_basics, read_index_datablockref}; - use crate::archeng::{read_datafile_header, EPICS_EPOCH_OFFSET}; + use crate::archeng::{ + open_read, read_channel, read_data_1, read_datafile_header, read_file_basics, read_index_datablockref, + StatsChannel, EPICS_EPOCH_OFFSET, + }; use err::Error; - use netpod::log::*; use netpod::timeunits::*; use netpod::FilePos; use netpod::Nanos; + use netpod::{log::*, NanoRange}; use std::path::PathBuf; /* @@ -1135,8 +1462,9 @@ mod test { #[test] fn read_file_basic_info() -> Result<(), Error> { let fut = async { - let mut f1 = open_read(CHN_0_MASTER_INDEX.into()).await?; - let res = read_file_basics(&mut f1).await?; + let stats = &StatsChannel::dummy(); + let mut f1 = open_read(CHN_0_MASTER_INDEX.into(), stats).await?; + let res = read_file_basics(&mut f1, stats).await?; assert_eq!(res.version, 3); assert_eq!(res.name_hash_anchor_beg, 88); assert_eq!(res.name_hash_anchor_len, 1009); @@ -1155,9 +1483,10 @@ mod test { #[test] fn read_for_channel() -> Result<(), Error> { let fut = async { - let mut index_file = open_read(CHN_0_MASTER_INDEX.into()).await?; + let stats = &StatsChannel::dummy(); + let mut index_file = open_read(CHN_0_MASTER_INDEX.into(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; - let res = read_channel(&mut index_file, channel_name).await?; + let res = read_channel(&mut index_file, channel_name, stats).await?; assert_eq!(res.is_some(), true); let res = res.unwrap(); assert_eq!(res.channel_name, channel_name); @@ -1177,14 +1506,15 @@ mod test { RTreeNodeRecord { ts1: 970417979635219603, ts2: 970429859806669835, child_or_id: 185015 }, */ let fut = async { + let stats = &StatsChannel::dummy(); let index_path: PathBuf = CHN_0_MASTER_INDEX.into(); - let mut index_file = open_read(index_path.clone()).await?; + let mut index_file = open_read(index_path.clone(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; const T0: u64 = 970351442331056677 + 1 + EPICS_EPOCH_OFFSET; let beg = Nanos { ns: T0 }; - let res = read_channel(&mut index_file, channel_name).await?; + let res = read_channel(&mut index_file, channel_name, stats).await?; let cib = res.unwrap(); - let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg).await?; + let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?; assert_eq!(res.is_some(), true); let res = res.unwrap(); assert_eq!(res.node.is_leaf, true); @@ -1195,7 +1525,7 @@ mod test { assert_eq!(rec.ts2.ns, 970417919634086480 + EPICS_EPOCH_OFFSET); assert_eq!(rec.child_or_id, 185074); let pos = FilePos { pos: rec.child_or_id }; - let datablock = read_index_datablockref(&mut index_file, pos).await?; + let datablock = read_index_datablockref(&mut index_file, pos, stats).await?; assert_eq!(datablock.data_header_pos, 52787); assert_eq!(datablock.fname, "20201001/20201001"); // The actual datafile for that time was not retained any longer. @@ -1208,14 +1538,19 @@ mod test { #[test] fn search_record_data() -> Result<(), Error> { let fut = async { + let stats = &StatsChannel::dummy(); let index_path: PathBuf = CHN_0_MASTER_INDEX.into(); - let mut index_file = open_read(index_path.clone()).await?; + let mut index_file = open_read(index_path.clone(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; const T0: u64 = 1002000000 * SEC + EPICS_EPOCH_OFFSET; let beg = Nanos { ns: T0 }; - let res = read_channel(&mut index_file, channel_name).await?; + let range = NanoRange { + beg: beg.ns, + end: beg.ns + 20 * SEC, + }; + let res = read_channel(&mut index_file, channel_name, stats).await?; let cib = res.unwrap(); - let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg).await?; + let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?; assert_eq!(res.is_some(), true); let res = res.unwrap(); assert_eq!(res.node.is_leaf, true); @@ -1226,13 +1561,13 @@ mod test { assert_eq!(rec.ts2.ns, 1002009299596362122 + EPICS_EPOCH_OFFSET); assert_eq!(rec.child_or_id, 2501903); let pos = FilePos { pos: rec.child_or_id }; - let datablock = read_index_datablockref(&mut index_file, pos).await?; + let datablock = read_index_datablockref(&mut index_file, pos, stats).await?; assert_eq!(datablock.data_header_pos().0, 9311367); assert_eq!(datablock.file_name(), "20211001/20211001"); let data_path = index_path.parent().unwrap().join(datablock.file_name()); - let mut data_file = open_read(data_path).await?; - let datafile_header = read_datafile_header(&mut data_file, datablock.data_header_pos()).await?; - let events = read_data_1(&mut data_file, &datafile_header).await?; + let mut data_file = open_read(data_path, stats).await?; + let datafile_header = read_datafile_header(&mut data_file, datablock.data_header_pos(), stats).await?; + let events = read_data_1(&mut data_file, &datafile_header, range.clone(), false, stats).await?; info!("read events: {:?}", events); Ok(()) }; @@ -1243,13 +1578,14 @@ mod test { #[test] fn search_record_at_beg() -> Result<(), Error> { let fut = async { - let mut index_file = open_read(CHN_0_MASTER_INDEX.into()).await?; + let stats = &StatsChannel::dummy(); + let mut index_file = open_read(CHN_0_MASTER_INDEX.into(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; const T0: u64 = 965081099942616289 + EPICS_EPOCH_OFFSET; let beg = Nanos { ns: T0 }; - let res = read_channel(&mut index_file, channel_name).await?; + let res = read_channel(&mut index_file, channel_name, stats).await?; let cib = res.unwrap(); - let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg).await?; + let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?; assert_eq!(res.is_some(), true); let res = res.unwrap(); assert_eq!(res.node.is_leaf, true); @@ -1264,13 +1600,14 @@ mod test { #[test] fn search_record_at_end() -> Result<(), Error> { let fut = async { - let mut index_file = open_read(CHN_0_MASTER_INDEX.into()).await?; + let stats = &StatsChannel::dummy(); + let mut index_file = open_read(CHN_0_MASTER_INDEX.into(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; const T0: u64 = 1002441959876114632 - 1 + EPICS_EPOCH_OFFSET; let beg = Nanos { ns: T0 }; - let res = read_channel(&mut index_file, channel_name).await?; + let res = read_channel(&mut index_file, channel_name, stats).await?; let cib = res.unwrap(); - let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg).await?; + let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?; assert_eq!(res.is_some(), true); let res = res.unwrap(); assert_eq!(res.node.pos.pos, 1861178); @@ -1284,13 +1621,14 @@ mod test { #[test] fn search_record_beyond_end() -> Result<(), Error> { let fut = async { - let mut index_file = open_read(CHN_0_MASTER_INDEX.into()).await?; + let stats = &StatsChannel::dummy(); + let mut index_file = open_read(CHN_0_MASTER_INDEX.into(), stats).await?; let channel_name = "X05DA-FE-WI1:TC1"; const T0: u64 = 1002441959876114632 - 0 + EPICS_EPOCH_OFFSET; let beg = Nanos { ns: T0 }; - let res = read_channel(&mut index_file, channel_name).await?; + let res = read_channel(&mut index_file, channel_name, stats).await?; let cib = res.unwrap(); - let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg).await?; + let (res, _stats) = search_record(&mut index_file, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?; assert_eq!(res.is_none(), true); Ok(()) }; diff --git a/archapp/src/archeng/datablockstream.rs b/archapp/src/archeng/datablockstream.rs index fbca50c..797d268 100644 --- a/archapp/src/archeng/datablockstream.rs +++ b/archapp/src/archeng/datablockstream.rs @@ -1,17 +1,21 @@ use crate::archeng::{ - open_read, read_channel, read_data_1, read_datafile_header, read_index_datablockref, search_record, + index_file_path_list, open_read, read_channel, read_data_1, read_datafile_header, read_index_datablockref, + search_record, search_record_expand, StatsChannel, }; -use crate::EventsItem; +use crate::eventsitem::EventsItem; +use crate::storagemerge::StorageMerge; +use crate::timed::Timed; use async_channel::{Receiver, Sender}; use err::Error; use futures_core::{Future, Stream}; use futures_util::{FutureExt, StreamExt}; -use items::{RangeCompletableItem, Sitemty, StreamItem, WithLen}; +use items::{inspect_timestamps, RangeCompletableItem, Sitemty, StreamItem, WithLen}; use netpod::{log::*, DataHeaderPos, FilePos, Nanos}; use netpod::{Channel, NanoRange}; use std::collections::VecDeque; use std::path::PathBuf; use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::{Context, Poll}; type FR = (Option>, Box); @@ -20,77 +24,133 @@ trait FretCb { fn call(&mut self, stream: &mut Pin<&mut DatablockStream>); } -async fn datablock_stream( - range: NanoRange, - channel: Channel, - base_dirs: VecDeque, - expand: bool, - tx: Sender>, -) { - match datablock_stream_inner(range, channel, base_dirs, expand, tx.clone()).await { - Ok(_) => {} - Err(e) => match tx.send(Err(e)).await { - Ok(_) => {} - Err(e) => { - if false { - error!("can not send. error: {}", e); - } - } - }, +static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0); + +fn channel_send_error() { + let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel); + if c < 10 { + error!("CHANNEL_SEND_ERROR {}", c); } } -async fn datablock_stream_inner( +async fn datablock_stream( range: NanoRange, channel: Channel, - base_dirs: VecDeque, + index_files_index_path: PathBuf, + _base_dirs: VecDeque, expand: bool, tx: Sender>, + max_events: u64, +) { + match datablock_stream_inner(range, channel, expand, index_files_index_path, tx.clone(), max_events).await { + Ok(_) => {} + Err(e) => { + if let Err(_) = tx.send(Err(e)).await { + channel_send_error(); + } + } + } +} + +async fn datablock_stream_inner_single_index( + range: NanoRange, + channel: Channel, + index_path: PathBuf, + expand: bool, + tx: Sender>, + max_events: u64, ) -> Result<(), Error> { - let basename = channel - .name() - .split("-") - .next() - .ok_or(Error::with_msg_no_trace("can not find base for channel"))?; - for base in base_dirs { - debug!( - "search for {:?} with basename: {} in path {:?}", - channel, basename, base - ); - // TODO need to try both: - let index_path = base.join(format!("archive_{}_SH", basename)).join("index"); - let res = open_read(index_path.clone()).await; - debug!("tried to open index file: {:?}", res); - if let Ok(mut index_file) = res { - if let Some(basics) = read_channel(&mut index_file, channel.name()).await? { + let mut events_tot = 0; + let stats = &StatsChannel::new(tx.clone()); + debug!("try to open index file: {:?}", index_path); + let res = open_read(index_path.clone(), stats).await; + debug!("opened index file: {:?} {:?}", index_path, res); + match res { + Ok(mut index_file) => { + if let Some(basics) = read_channel(&mut index_file, channel.name(), stats).await? { let beg = Nanos { ns: range.beg }; + let mut expand_beg = expand; + let mut index_ts_max = 0; let mut search_ts = beg.clone(); let mut last_data_file_path = PathBuf::new(); let mut last_data_file_pos = DataHeaderPos(0); loop { - // TODO for expand mode, this needs another search function. - let (res, _stats) = - search_record(&mut index_file, basics.rtree_m, basics.rtree_start_pos, search_ts).await?; + let timed_search = Timed::new("search next record"); + let (res, _stats) = if expand_beg { + // TODO even though this is an entry in the index, it may reference + // non-existent blocks. + // Therefore, lower expand_beg flag at some later stage only if we've really + // found at least one event in the block. + expand_beg = false; + search_record_expand( + &mut index_file, + basics.rtree_m, + basics.rtree_start_pos, + search_ts, + stats, + ) + .await? + } else { + search_record( + &mut index_file, + basics.rtree_m, + basics.rtree_start_pos, + search_ts, + stats, + ) + .await? + }; + drop(timed_search); if let Some(nrec) = res { let rec = nrec.rec(); trace!("found record: {:?}", rec); let pos = FilePos { pos: rec.child_or_id }; // TODO rename Datablock? → IndexNodeDatablock trace!("READ Datablock FROM {:?}\n", pos); - let datablock = read_index_datablockref(&mut index_file, pos).await?; + let datablock = read_index_datablockref(&mut index_file, pos, stats).await?; trace!("Datablock: {:?}\n", datablock); let data_path = index_path.parent().unwrap().join(datablock.file_name()); if data_path == last_data_file_path && datablock.data_header_pos() == last_data_file_pos { debug!("skipping because it is the same block"); } else { trace!("try to open data_path: {:?}", data_path); - match open_read(data_path.clone()).await { + match open_read(data_path.clone(), stats).await { Ok(mut data_file) => { let datafile_header = - read_datafile_header(&mut data_file, datablock.data_header_pos()).await?; + read_datafile_header(&mut data_file, datablock.data_header_pos(), stats) + .await?; trace!("datafile_header -------------- HEADER\n{:?}", datafile_header); - let events = read_data_1(&mut data_file, &datafile_header).await?; + let events = + read_data_1(&mut data_file, &datafile_header, range.clone(), expand_beg, stats) + .await?; + if false { + let msg = inspect_timestamps(&events, range.clone()); + trace!("datablock_stream_inner_single_index read_data_1\n{}", msg); + } + { + let mut ts_max = 0; + use items::WithTimestamps; + for i in 0..events.len() { + let ts = events.ts(i); + if ts < ts_max { + error!("unordered event within block at ts {}", ts); + break; + } else { + ts_max = ts; + } + if ts < index_ts_max { + error!( + "unordered event in index branch ts {} index_ts_max {}", + ts, index_ts_max + ); + break; + } else { + index_ts_max = ts; + } + } + } trace!("Was able to read data: {} events", events.len()); + events_tot += events.len() as u64; let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(events))); tx.send(item).await?; } @@ -111,12 +171,59 @@ async fn datablock_stream_inner( warn!("nothing found, break"); break; } + if events_tot >= max_events { + warn!("reached events_tot {} max_events {}", events_tot, max_events); + break; + } } + } else { + warn!("can not read channel basics from {:?}", index_path); } - } else { + Ok(()) + } + Err(e) => { warn!("can not find index file at {:?}", index_path); + Err(Error::with_msg_no_trace(format!("can not open index file: {}", e))) } } +} + +async fn datablock_stream_inner( + range: NanoRange, + channel: Channel, + expand: bool, + index_files_index_path: PathBuf, + tx: Sender>, + max_events: u64, +) -> Result<(), Error> { + let stats = &StatsChannel::new(tx.clone()); + let index_file_path_list = index_file_path_list(channel.clone(), index_files_index_path, stats).await?; + let mut inner_rxs = vec![]; + let mut names = vec![]; + for index_path in index_file_path_list { + let (tx, rx) = async_channel::bounded(2); + let task = datablock_stream_inner_single_index( + range.clone(), + channel.clone(), + (&index_path).into(), + expand, + tx, + max_events, + ); + taskrun::spawn(task); + inner_rxs.push(Box::pin(rx) as Pin> + Send>>); + names.push(index_path.to_str().unwrap().into()); + } + let task = async move { + let mut inp = StorageMerge::new(inner_rxs, names, range.clone()); + while let Some(k) = inp.next().await { + if let Err(_) = tx.send(k).await { + channel_send_error(); + break; + } + } + }; + taskrun::spawn(task); Ok(()) } @@ -132,14 +239,22 @@ pub struct DatablockStream { } impl DatablockStream { - pub fn for_channel_range(range: NanoRange, channel: Channel, base_dirs: VecDeque, expand: bool) -> Self { + pub fn for_channel_range( + range: NanoRange, + channel: Channel, + base_dirs: VecDeque, + expand: bool, + max_events: u64, + ) -> Self { let (tx, rx) = async_channel::bounded(1); taskrun::spawn(datablock_stream( range.clone(), channel.clone(), + "/index/c5mapped".into(), base_dirs.clone(), expand.clone(), tx, + max_events, )); let ret = Self { range, @@ -151,6 +266,10 @@ impl DatablockStream { done: false, complete: false, }; + // TODO keeping for compatibility at the moment: + let _ = &ret.range; + let _ = &ret.channel; + let _ = &ret.expand; ret } @@ -169,7 +288,7 @@ impl DatablockStream { (None, Box::new(Cb {})) } - async fn start_with_base_dir(path: PathBuf) -> FR { + async fn start_with_base_dir(_path: PathBuf) -> FR { warn!("start_with_base_dir"); struct Cb {} impl FretCb for Cb { @@ -199,9 +318,7 @@ impl Stream for DatablockStream { } else if self.done { self.complete = true; Ready(None) - } else if true { - self.rx.poll_next_unpin(cx) - } else { + } else if false { match self.fut.poll_unpin(cx) { Ready((k, mut fr)) => { fr.call(&mut self); @@ -212,6 +329,8 @@ impl Stream for DatablockStream { } Pending => Pending, } + } else { + self.rx.poll_next_unpin(cx) }; } } @@ -219,9 +338,8 @@ impl Stream for DatablockStream { #[cfg(test)] mod test { - use crate::EventsItem; - use super::DatablockStream; + use crate::eventsitem::EventsItem; use chrono::{DateTime, Utc}; use err::Error; use futures_util::StreamExt; @@ -258,7 +376,7 @@ mod test { .map(PathBuf::from) .collect(); let expand = false; - let datablocks = DatablockStream::for_channel_range(range.clone(), channel, base_dirs, expand); + let datablocks = DatablockStream::for_channel_range(range.clone(), channel, base_dirs, expand, u64::MAX); let filtered = RangeFilter::<_, EventsItem>::new(datablocks, range, expand); let mut stream = filtered; while let Some(block) = stream.next().await { diff --git a/archapp/src/archeng/datastream.rs b/archapp/src/archeng/datastream.rs deleted file mode 100644 index 7b5d56f..0000000 --- a/archapp/src/archeng/datastream.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::EventsItem; -use futures_core::Stream; -use items::Sitemty; -use std::pin::Pin; -use std::task::{Context, Poll}; - -pub struct DataStream {} - -impl Stream for DataStream { - type Item = Sitemty; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - let _ = self; - let _ = cx; - todo!() - } -} diff --git a/archapp/src/archeng/indexfiles.rs b/archapp/src/archeng/indexfiles.rs new file mode 100644 index 0000000..c0ea2ee --- /dev/null +++ b/archapp/src/archeng/indexfiles.rs @@ -0,0 +1,312 @@ +use crate::wrap_task; +use async_channel::Receiver; +use err::Error; +use futures_core::Future; +use futures_core::Stream; +use futures_util::stream::unfold; +use futures_util::FutureExt; +use netpod::log::*; +use netpod::ChannelArchiver; +use netpod::Database; +use std::path::PathBuf; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::fs::read_dir; +use tokio_postgres::Client as PgClient; + +pub fn list_index_files(node: &ChannelArchiver) -> Receiver> { + let node = node.clone(); + let (tx, rx) = async_channel::bounded(4); + let tx2 = tx.clone(); + let task = async move { + for bp in &node.data_base_paths { + let mut rd = read_dir(bp).await?; + while let Some(e) = rd.next_entry().await? { + let ft = e.file_type().await?; + if ft.is_dir() { + let mut rd = read_dir(e.path()).await?; + while let Some(e) = rd.next_entry().await? { + let ft = e.file_type().await?; + if false && ft.is_dir() { + let mut rd = read_dir(e.path()).await?; + while let Some(e) = rd.next_entry().await? { + let ft = e.file_type().await?; + if ft.is_file() { + if e.file_name().to_string_lossy() == "index" { + tx.send(Ok(e.path())).await?; + } + } + } + } else if ft.is_file() { + if e.file_name().to_string_lossy() == "index" { + tx.send(Ok(e.path())).await?; + } + } + } + } else if ft.is_file() { + if e.file_name().to_string_lossy() == "index" { + tx.send(Ok(e.path())).await?; + } + } + } + } + Ok::<_, Error>(()) + }; + wrap_task(task, tx2); + rx +} + +pub struct ScanIndexFiles0 {} + +impl Stream for ScanIndexFiles0 { + type Item = (); + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let _ = cx; + todo!() + } +} + +pub async fn get_level_0(conf: ChannelArchiver) -> Result, Error> { + let mut ret = vec![]; + for bp in &conf.data_base_paths { + let mut rd = read_dir(bp).await?; + while let Some(e) = rd.next_entry().await? { + if e.file_name().to_string_lossy().contains("index") { + warn!("Top-level data path contains `index` entry"); + } + let ft = e.file_type().await?; + if ft.is_dir() { + ret.push(e.path()); + } + } + } + Ok(ret) +} + +pub async fn get_level_1(lev0: Vec) -> Result, Error> { + let mut ret = vec![]; + for bp in lev0 { + let mut rd = read_dir(bp).await?; + while let Some(e) = rd.next_entry().await? { + let ft = e.file_type().await?; + if ft.is_file() { + if e.file_name().to_string_lossy() == "index" { + ret.push(e.path()); + } + } + } + } + Ok(ret) +} + +pub async fn database_connect(db_config: &Database) -> Result { + let d = db_config; + let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, 5432, d.name); + let (cl, conn) = tokio_postgres::connect(&uri, tokio_postgres::NoTls).await?; + // TODO monitor connection drop. + let _cjh = tokio::spawn(async move { + if let Err(e) = conn.await { + error!("connection error: {}", e); + } + Ok::<_, Error>(()) + }); + Ok(cl) +} + +pub trait UnfoldExec { + type Output: Send; + fn exec(self) -> Pin, Error>> + Send>> + where + Self: Sized; +} + +pub fn unfold_stream(st: St) -> impl Stream> +where + St: UnfoldExec + Send, + T: Send, +{ + enum UnfoldState { + Running(St), + Done, + } + unfold(UnfoldState::Running(st), |st| async move { + match st { + UnfoldState::Running(st) => match st.exec().await { + Ok(Some((item, st))) => Some((Ok(item), UnfoldState::Running(st))), + Ok(None) => None, + Err(e) => Some((Err(e), UnfoldState::Done)), + }, + UnfoldState::Done => None, + } + }) +} + +enum ScanIndexFilesSteps { + Level0, + Level1(Vec), + Done, +} + +struct ScanIndexFiles { + conf: ChannelArchiver, + steps: ScanIndexFilesSteps, +} + +impl ScanIndexFiles { + fn new(conf: ChannelArchiver) -> Self { + Self { + conf, + steps: ScanIndexFilesSteps::Level0, + } + } + + async fn exec(mut self) -> Result, Error> { + match self.steps { + ScanIndexFilesSteps::Level0 => { + let res = get_level_0(self.conf.clone()).await?; + self.steps = ScanIndexFilesSteps::Level1(res); + let item = format!("level 0 done"); + Ok(Some((item, self))) + } + ScanIndexFilesSteps::Level1(paths) => { + let paths = get_level_1(paths).await?; + info!("collected {} level 1 paths", paths.len()); + let dbc = database_connect(&self.conf.database).await?; + for p in paths { + let sql = "insert into indexfiles (path) values ($1) on conflict do nothing"; + dbc.query(sql, &[&p.to_string_lossy()]).await?; + } + self.steps = ScanIndexFilesSteps::Done; + let item = format!("level 1 done"); + Ok(Some((item, self))) + } + ScanIndexFilesSteps::Done => Ok(None), + } + } +} + +impl UnfoldExec for ScanIndexFiles { + type Output = String; + + fn exec(self) -> Pin, Error>> + Send>> + where + Self: Sized, + { + Box::pin(self.exec()) + } +} + +pub fn scan_index_files(conf: ChannelArchiver) -> impl Stream> { + unfold_stream(ScanIndexFiles::new(conf.clone())) + /* + enum UnfoldState { + Running(ScanIndexFiles), + Done, + } + unfold(UnfoldState::Running(ScanIndexFiles::new(conf)), |st| async move { + match st { + UnfoldState::Running(st) => match st.exec().await { + Ok(Some((item, st))) => Some((Ok(item), UnfoldState::Running(st))), + Ok(None) => None, + Err(e) => { + error!("{}", e); + Some((Err(e), UnfoldState::Done)) + } + }, + UnfoldState::Done => None, + } + }) + */ +} + +pub fn unfold1() -> impl Stream { + unfold(123u32, |st| async move { Some((format!("{}", st), st)) }) +} + +pub fn unfold2(_conf: ChannelArchiver) -> () { + /*let f1 = async move { + let _list = get_level_0(conf).await?; + let yld = format!("level 0 done"); + let fut = async { Ok(None) }; + Ok(Some((yld, Box::pin(fut)))) + }; + unfold( + Box::pin(f1) as Pin, Error>>>>, + |st| async { + match st.await { + Ok(None) => None, + Ok(Some((item, st))) => { + //Some((item, st)); + //Some((String::new(), Box::pin(async { Ok(None) }))) + None + } + Err(e) => { + error!("{}", e); + None + } + } + }, + )*/ + err::todoval() +} + +// ------------------------------------------------- + +enum ScanChannelsSteps { + Start, + SelectIndexFile, + Done, +} + +struct ScanChannels { + conf: ChannelArchiver, + steps: ScanChannelsSteps, +} + +impl ScanChannels { + fn new(conf: ChannelArchiver) -> Self { + Self { + conf, + steps: ScanChannelsSteps::Start, + } + } + + async fn exec(mut self) -> Result, Error> { + use ScanChannelsSteps::*; + match self.steps { + Start => { + self.steps = SelectIndexFile; + Ok(Some((format!("Start"), self))) + } + SelectIndexFile => { + let dbc = database_connect(&self.conf.database).await?; + let sql = + "select path from indexfiles where ts_last_channel_search < now() - interval '1 hour' limit 1"; + let rows = dbc.query(sql, &[]).await?; + let mut paths = vec![]; + for row in rows { + paths.push(row.get::<_, String>(0)); + } + self.steps = Done; + Ok(Some((format!("SelectIndexFile {:?}", paths), self))) + } + Done => Ok(None), + } + } +} + +impl UnfoldExec for ScanChannels { + type Output = String; + + fn exec(self) -> Pin, Error>> + Send>> + where + Self: Sized, + { + Box::pin(self.exec()) + } +} + +pub fn scan_channels(conf: ChannelArchiver) -> impl Stream> { + unfold_stream(ScanChannels::new(conf.clone())) +} diff --git a/archapp/src/archeng/pipe.rs b/archapp/src/archeng/pipe.rs index 5368f35..6d29806 100644 --- a/archapp/src/archeng/pipe.rs +++ b/archapp/src/archeng/pipe.rs @@ -4,6 +4,7 @@ use err::Error; use futures_core::Stream; use futures_util::StreamExt; use items::Framable; +use netpod::ChannelConfigQuery; use netpod::{query::RawEventsQuery, ChannelArchiver}; use std::pin::Pin; use streams::rangefilter::RangeFilter; @@ -12,27 +13,35 @@ pub async fn make_event_pipe( evq: &RawEventsQuery, conf: &ChannelArchiver, ) -> Result> + Send>>, Error> { - // In order to extract something from the channel, need to look up first the type of the channel. - //let ci = channel_info(&evq.channel, aa).await?; - /*let mut inps = vec![]; - for p1 in &aa.data_base_paths { - let p2 = p1.clone(); - let p3 = make_single_event_pipe(evq, p2).await?; - inps.push(p3); - } - let sm = StorageMerge { - inprng: inps.len() - 1, - current_inp_item: (0..inps.len()).into_iter().map(|_| None).collect(), - completed_inps: vec![false; inps.len()], - inps, - };*/ let range = evq.range.clone(); let channel = evq.channel.clone(); let expand = evq.agg_kind.need_expand(); - let data = DatablockStream::for_channel_range(range.clone(), channel, conf.data_base_paths.clone().into(), expand); + + // TODO I need the numeric type here which I expect for that channel in order to construct FrameMaker. + // TODO Need to pass that requirement down to disk reader: error if type changes. + + let channel_config = { + let q = ChannelConfigQuery { + channel: channel.clone(), + range: range.clone(), + }; + crate::archeng::channel_config(&q, conf).await? + }; + + let data = DatablockStream::for_channel_range( + range.clone(), + channel, + conf.data_base_paths.clone().into(), + expand, + u64::MAX, + ); let filtered = RangeFilter::new(data, range, expand); let stream = filtered; - let mut frame_maker = Box::new(FrameMaker::untyped(evq.agg_kind.clone())) as Box; + let mut frame_maker = Box::new(FrameMaker::with_item_type( + channel_config.scalar_type.clone(), + channel_config.shape.clone(), + evq.agg_kind.clone(), + )) as Box; let ret = stream.map(move |j| frame_maker.make_frame(j)); Ok(Box::pin(ret)) } diff --git a/archapp/src/binnedevents.rs b/archapp/src/binnedevents.rs new file mode 100644 index 0000000..596098a --- /dev/null +++ b/archapp/src/binnedevents.rs @@ -0,0 +1,465 @@ +use items::{ + xbinnedscalarevents::XBinnedScalarEvents, xbinnedwaveevents::XBinnedWaveEvents, Appendable, Clearable, + PushableIndex, WithLen, WithTimestamps, +}; +use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape}; + +use crate::{ + eventsitem::EventsItem, + plainevents::{PlainEvents, ScalarPlainEvents}, +}; + +#[derive(Debug)] +pub enum SingleBinWaveEvents { + Byte(XBinnedScalarEvents), + Short(XBinnedScalarEvents), + Int(XBinnedScalarEvents), + Float(XBinnedScalarEvents), + Double(XBinnedScalarEvents), +} + +impl SingleBinWaveEvents { + pub fn variant_name(&self) -> String { + use SingleBinWaveEvents::*; + match self { + Byte(_) => format!("Byte"), + Short(_) => format!("Short"), + Int(_) => format!("Int"), + Float(_) => format!("Float"), + Double(_) => format!("Double"), + } + } + + fn x_aggregate(self, ak: &AggKind) -> EventsItem { + use SingleBinWaveEvents::*; + match self { + Byte(k) => match ak { + AggKind::EventBlobs => panic!(), + AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::Byte(k))), + AggKind::TimeWeightedScalar => err::todoval(), + AggKind::DimXBins1 => err::todoval(), + AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), + }, + _ => err::todoval(), + } + } +} + +impl Clearable for SingleBinWaveEvents { + fn clear(&mut self) { + match self { + SingleBinWaveEvents::Byte(k) => k.clear(), + SingleBinWaveEvents::Short(k) => k.clear(), + SingleBinWaveEvents::Int(k) => k.clear(), + SingleBinWaveEvents::Float(k) => k.clear(), + SingleBinWaveEvents::Double(k) => k.clear(), + } + } +} + +impl Appendable for SingleBinWaveEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Byte(k) => Self::Byte(k.empty_like_self()), + Self::Short(k) => Self::Short(k.empty_like_self()), + Self::Int(k) => Self::Int(k.empty_like_self()), + Self::Float(k) => Self::Float(k.empty_like_self()), + Self::Double(k) => Self::Double(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.append(j), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.append(j), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.append(j), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.append(j), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for SingleBinWaveEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for SingleBinWaveEvents { + fn len(&self) -> usize { + use SingleBinWaveEvents::*; + match self { + Byte(j) => j.len(), + Short(j) => j.len(), + Int(j) => j.len(), + Float(j) => j.len(), + Double(j) => j.len(), + } + } +} + +impl WithTimestamps for SingleBinWaveEvents { + fn ts(&self, ix: usize) -> u64 { + use SingleBinWaveEvents::*; + match self { + Byte(j) => j.ts(ix), + Short(j) => j.ts(ix), + Int(j) => j.ts(ix), + Float(j) => j.ts(ix), + Double(j) => j.ts(ix), + } + } +} + +impl HasShape for SingleBinWaveEvents { + fn shape(&self) -> Shape { + use SingleBinWaveEvents::*; + match self { + Byte(_) => Shape::Scalar, + Short(_) => Shape::Scalar, + Int(_) => Shape::Scalar, + Float(_) => Shape::Scalar, + Double(_) => Shape::Scalar, + } + } +} + +impl HasScalarType for SingleBinWaveEvents { + fn scalar_type(&self) -> ScalarType { + use SingleBinWaveEvents::*; + match self { + Byte(_) => ScalarType::I8, + Short(_) => ScalarType::I16, + Int(_) => ScalarType::I32, + Float(_) => ScalarType::F32, + Double(_) => ScalarType::F64, + } + } +} + +#[derive(Debug)] +pub enum MultiBinWaveEvents { + Byte(XBinnedWaveEvents), + Short(XBinnedWaveEvents), + Int(XBinnedWaveEvents), + Float(XBinnedWaveEvents), + Double(XBinnedWaveEvents), +} + +impl MultiBinWaveEvents { + pub fn variant_name(&self) -> String { + use MultiBinWaveEvents::*; + match self { + Byte(_) => format!("Byte"), + Short(_) => format!("Short"), + Int(_) => format!("Int"), + Float(_) => format!("Float"), + Double(_) => format!("Double"), + } + } + + fn x_aggregate(self, ak: &AggKind) -> EventsItem { + use MultiBinWaveEvents::*; + match self { + Byte(k) => match ak { + AggKind::EventBlobs => panic!(), + AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::MultiBinWave(MultiBinWaveEvents::Byte(k))), + AggKind::TimeWeightedScalar => err::todoval(), + AggKind::DimXBins1 => err::todoval(), + AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), + }, + _ => err::todoval(), + } + } +} + +impl Clearable for MultiBinWaveEvents { + fn clear(&mut self) { + match self { + MultiBinWaveEvents::Byte(k) => k.clear(), + MultiBinWaveEvents::Short(k) => k.clear(), + MultiBinWaveEvents::Int(k) => k.clear(), + MultiBinWaveEvents::Float(k) => k.clear(), + MultiBinWaveEvents::Double(k) => k.clear(), + } + } +} + +impl Appendable for MultiBinWaveEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Byte(k) => Self::Byte(k.empty_like_self()), + Self::Short(k) => Self::Short(k.empty_like_self()), + Self::Int(k) => Self::Int(k.empty_like_self()), + Self::Float(k) => Self::Float(k.empty_like_self()), + Self::Double(k) => Self::Double(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.append(j), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.append(j), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.append(j), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.append(j), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for MultiBinWaveEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for MultiBinWaveEvents { + fn len(&self) -> usize { + use MultiBinWaveEvents::*; + match self { + Byte(j) => j.len(), + Short(j) => j.len(), + Int(j) => j.len(), + Float(j) => j.len(), + Double(j) => j.len(), + } + } +} + +impl WithTimestamps for MultiBinWaveEvents { + fn ts(&self, ix: usize) -> u64 { + use MultiBinWaveEvents::*; + match self { + Byte(j) => j.ts(ix), + Short(j) => j.ts(ix), + Int(j) => j.ts(ix), + Float(j) => j.ts(ix), + Double(j) => j.ts(ix), + } + } +} + +impl HasShape for MultiBinWaveEvents { + fn shape(&self) -> Shape { + use MultiBinWaveEvents::*; + match self { + Byte(_) => Shape::Scalar, + Short(_) => Shape::Scalar, + Int(_) => Shape::Scalar, + Float(_) => Shape::Scalar, + Double(_) => Shape::Scalar, + } + } +} + +impl HasScalarType for MultiBinWaveEvents { + fn scalar_type(&self) -> ScalarType { + use MultiBinWaveEvents::*; + match self { + Byte(_) => ScalarType::I8, + Short(_) => ScalarType::I16, + Int(_) => ScalarType::I32, + Float(_) => ScalarType::F32, + Double(_) => ScalarType::F64, + } + } +} + +#[derive(Debug)] +pub enum XBinnedEvents { + Scalar(ScalarPlainEvents), + SingleBinWave(SingleBinWaveEvents), + MultiBinWave(MultiBinWaveEvents), +} + +impl XBinnedEvents { + pub fn variant_name(&self) -> String { + use XBinnedEvents::*; + match self { + Scalar(h) => format!("Scalar({})", h.variant_name()), + SingleBinWave(h) => format!("SingleBinWave({})", h.variant_name()), + MultiBinWave(h) => format!("MultiBinWave({})", h.variant_name()), + } + } + + pub fn x_aggregate(self, ak: &AggKind) -> EventsItem { + use XBinnedEvents::*; + match self { + Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)), + SingleBinWave(k) => k.x_aggregate(ak), + MultiBinWave(k) => k.x_aggregate(ak), + } + } +} + +impl Clearable for XBinnedEvents { + fn clear(&mut self) { + match self { + XBinnedEvents::Scalar(k) => k.clear(), + XBinnedEvents::SingleBinWave(k) => k.clear(), + XBinnedEvents::MultiBinWave(k) => k.clear(), + } + } +} + +impl Appendable for XBinnedEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Scalar(k) => Self::Scalar(k.empty_like_self()), + Self::SingleBinWave(k) => Self::SingleBinWave(k.empty_like_self()), + Self::MultiBinWave(k) => Self::MultiBinWave(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Scalar(k) => match src { + Self::Scalar(j) => k.append(j), + _ => panic!(), + }, + Self::SingleBinWave(k) => match src { + Self::SingleBinWave(j) => k.append(j), + _ => panic!(), + }, + Self::MultiBinWave(k) => match src { + Self::MultiBinWave(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for XBinnedEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Scalar(k) => match src { + Self::Scalar(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::SingleBinWave(k) => match src { + Self::SingleBinWave(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::MultiBinWave(k) => match src { + Self::MultiBinWave(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for XBinnedEvents { + fn len(&self) -> usize { + use XBinnedEvents::*; + match self { + Scalar(j) => j.len(), + SingleBinWave(j) => j.len(), + MultiBinWave(j) => j.len(), + } + } +} + +impl WithTimestamps for XBinnedEvents { + fn ts(&self, ix: usize) -> u64 { + use XBinnedEvents::*; + match self { + Scalar(j) => j.ts(ix), + SingleBinWave(j) => j.ts(ix), + MultiBinWave(j) => j.ts(ix), + } + } +} + +impl HasShape for XBinnedEvents { + fn shape(&self) -> Shape { + use XBinnedEvents::*; + match self { + Scalar(h) => h.shape(), + SingleBinWave(h) => h.shape(), + MultiBinWave(h) => h.shape(), + } + } +} + +impl HasScalarType for XBinnedEvents { + fn scalar_type(&self) -> ScalarType { + use XBinnedEvents::*; + match self { + Scalar(h) => h.scalar_type(), + SingleBinWave(h) => h.scalar_type(), + MultiBinWave(h) => h.scalar_type(), + } + } +} diff --git a/archapp/src/events.rs b/archapp/src/events.rs index 9287728..3888aba 100644 --- a/archapp/src/events.rs +++ b/archapp/src/events.rs @@ -1,9 +1,10 @@ +use crate::binnedevents::{MultiBinWaveEvents, SingleBinWaveEvents, XBinnedEvents}; +use crate::eventsitem::EventsItem; use crate::generated::EPICSEvent::PayloadType; use crate::parse::multi::parse_all_ts; use crate::parse::PbFileReader; -use crate::{ - EventsItem, MultiBinWaveEvents, PlainEvents, ScalarPlainEvents, SingleBinWaveEvents, WavePlainEvents, XBinnedEvents, -}; +use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents}; +use crate::storagemerge::StorageMerge; use chrono::{TimeZone, Utc}; use err::Error; use futures_core::Stream; @@ -22,7 +23,6 @@ use serde_json::Value as JsonValue; use std::io::SeekFrom; use std::path::PathBuf; use std::pin::Pin; -use std::task::{Context, Poll}; use tokio::fs::{read_dir, File}; use tokio::io::{AsyncReadExt, AsyncSeekExt}; @@ -51,125 +51,13 @@ pub fn parse_data_filename(s: &str) -> Result { Ok(ret) } -struct StorageMerge { - inps: Vec> + Send>>>, - completed_inps: Vec, - current_inp_item: Vec>, - inprng: usize, -} - -impl StorageMerge { - fn refill_if_needed(mut self: Pin<&mut Self>, cx: &mut Context) -> Result<(Pin<&mut Self>, bool), Error> { - use Poll::*; - let mut is_pending = false; - for i in 0..self.inps.len() { - if self.current_inp_item[i].is_none() && self.completed_inps[i] == false { - match self.inps[i].poll_next_unpin(cx) { - Ready(j) => { - // - match j { - Some(j) => match j { - Ok(j) => match j { - StreamItem::DataItem(j) => match j { - RangeCompletableItem::Data(j) => { - self.current_inp_item[i] = Some(j); - } - RangeCompletableItem::RangeComplete => {} - }, - StreamItem::Log(_) => {} - StreamItem::Stats(_) => {} - }, - Err(e) => { - self.completed_inps[i] = true; - error!("inp err {:?}", e); - } - }, - None => { - // - self.completed_inps[i] = true; - } - } - } - Pending => { - is_pending = true; - } - } - } - } - Ok((self, is_pending)) - } - - fn decide_next_item(&mut self) -> Result>, Error> { - let not_found = 999; - let mut i1 = self.inprng; - let mut j1 = not_found; - let mut tsmin = u64::MAX; - #[allow(unused)] - use items::{WithLen, WithTimestamps}; - loop { - if self.completed_inps[i1] { - } else { - match self.current_inp_item[i1].as_ref() { - None => panic!(), - Some(j) => { - if j.len() == 0 { - j1 = i1; - break; - } else { - let ts = j.ts(0); - if ts < tsmin { - tsmin = ts; - j1 = i1; - self.inprng = i1; - } else { - } - } - } - } - } - i1 -= 1; - if i1 == 0 { - break; - } - } - if j1 >= not_found { - Ok(None) - } else { - let j = self.current_inp_item[j1] - .take() - .map(|j| Ok(StreamItem::DataItem(RangeCompletableItem::Data(j)))); - Ok(j) - } - } -} - -impl Stream for StorageMerge { - type Item = Sitemty; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - use Poll::*; - let (mut self2, is_pending) = self.refill_if_needed(cx).unwrap(); - if is_pending { - Pending - } else { - match self2.decide_next_item() { - Ok(j) => Ready(j), - Err(e) => { - error!("impl Stream for StorageMerge {:?}", e); - panic!() - } - } - } - } -} - pub trait FrameMakerTrait: Send { fn make_frame(&mut self, ei: Sitemty) -> Box; } pub struct FrameMaker { - scalar_type: Option, - shape: Option, + scalar_type: ScalarType, + shape: Shape, agg_kind: AggKind, } @@ -184,19 +72,11 @@ impl FrameMaker { pub fn with_item_type(scalar_type: ScalarType, shape: Shape, agg_kind: AggKind) -> Self { Self { - scalar_type: Some(scalar_type), - shape: Some(shape), + scalar_type: scalar_type, + shape: shape, agg_kind: agg_kind, } } - - pub fn untyped(agg_kind: AggKind) -> Self { - Self { - scalar_type: None, - shape: None, - agg_kind, - } - } } #[allow(unused_macros)] @@ -351,82 +231,16 @@ macro_rules! arm1 { impl FrameMakerTrait for FrameMaker { fn make_frame(&mut self, item: Sitemty) -> Box { - // Take from `self` the expected inner type. - // If `ei` is not some data, then I can't dynamically determine the expected T of Sitemty. - // Therefore, I need to decide that based on given parameters. - // see also channel_info in this mod. - if self.scalar_type.is_none() || self.shape.is_none() { - //let scalar_type = &ScalarType::I8; - //let shape = &Shape::Scalar; - //let agg_kind = &self.agg_kind; - let (scalar_type, shape) = match &item { - Ok(k) => match k { - StreamItem::DataItem(k) => match k { - RangeCompletableItem::RangeComplete => (ScalarType::I8, Shape::Scalar), - RangeCompletableItem::Data(k) => match k { - EventsItem::Plain(k) => match k { - PlainEvents::Scalar(k) => match k { - ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar), - ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar), - ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar), - ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar), - ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar), - }, - PlainEvents::Wave(k) => match k { - WavePlainEvents::Byte(k) => (ScalarType::I8, Shape::Wave(k.vals[0].len() as u32)), - WavePlainEvents::Short(k) => (ScalarType::I16, Shape::Wave(k.vals[0].len() as u32)), - WavePlainEvents::Int(k) => (ScalarType::I32, Shape::Wave(k.vals[0].len() as u32)), - WavePlainEvents::Float(k) => (ScalarType::F32, Shape::Wave(k.vals[0].len() as u32)), - WavePlainEvents::Double(k) => { - (ScalarType::F64, Shape::Wave(k.vals[0].len() as u32)) - } - }, - }, - EventsItem::XBinnedEvents(k) => match k { - XBinnedEvents::Scalar(k) => match k { - ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar), - ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar), - ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar), - ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar), - ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar), - }, - XBinnedEvents::SingleBinWave(k) => match k { - SingleBinWaveEvents::Byte(_) => todo!(), - SingleBinWaveEvents::Short(_) => todo!(), - SingleBinWaveEvents::Int(_) => todo!(), - SingleBinWaveEvents::Float(_) => todo!(), - SingleBinWaveEvents::Double(_) => todo!(), - }, - XBinnedEvents::MultiBinWave(k) => match k { - MultiBinWaveEvents::Byte(_) => todo!(), - MultiBinWaveEvents::Short(_) => todo!(), - MultiBinWaveEvents::Int(_) => todo!(), - MultiBinWaveEvents::Float(_) => todo!(), - MultiBinWaveEvents::Double(_) => todo!(), - }, - }, - }, - }, - StreamItem::Log(_) => (ScalarType::I8, Shape::Scalar), - StreamItem::Stats(_) => (ScalarType::I8, Shape::Scalar), - }, - Err(_) => (ScalarType::I8, Shape::Scalar), - }; - self.scalar_type = Some(scalar_type); - self.shape = Some(shape); - } - { - let scalar_type = self.scalar_type.as_ref().unwrap(); - let shape = self.shape.as_ref().unwrap(); - let agg_kind = &self.agg_kind; - match scalar_type { - ScalarType::I8 => arm1!(item, i8, Byte, shape, agg_kind), - ScalarType::I16 => arm1!(item, i16, Short, shape, agg_kind), - ScalarType::I32 => arm1!(item, i32, Int, shape, agg_kind), - ScalarType::F32 => arm1!(item, f32, Float, shape, agg_kind), - ScalarType::F64 => arm1!(item, f64, Double, shape, agg_kind), - _ => err::todoval(), - } + let scalar_type = &self.scalar_type; + let shape = &self.shape; + let agg_kind = &self.agg_kind; + match scalar_type { + ScalarType::I8 => arm1!(item, i8, Byte, shape, agg_kind), + ScalarType::I16 => arm1!(item, i16, Short, shape, agg_kind), + ScalarType::I32 => arm1!(item, i32, Int, shape, agg_kind), + ScalarType::F32 => arm1!(item, f32, Float, shape, agg_kind), + ScalarType::F64 => arm1!(item, f64, Double, shape, agg_kind), + _ => err::todoval(), } } } @@ -437,17 +251,14 @@ pub async fn make_event_pipe( ) -> Result> + Send>>, Error> { let ci = channel_info(&evq.channel, aa).await?; let mut inps = vec![]; + let mut names = vec![]; for p1 in &aa.data_base_paths { let p2 = p1.clone(); let p3 = make_single_event_pipe(evq, p2).await?; inps.push(p3); + names.push(p1.to_str().unwrap().into()); } - let sm = StorageMerge { - inprng: inps.len() - 1, - current_inp_item: (0..inps.len()).into_iter().map(|_| None).collect(), - completed_inps: vec![false; inps.len()], - inps, - }; + let sm = StorageMerge::new(inps, names, evq.range.clone()); let mut frame_maker = Box::new(FrameMaker::with_item_type( ci.scalar_type.clone(), ci.shape.clone(), @@ -462,7 +273,6 @@ pub async fn make_single_event_pipe( base_path: PathBuf, ) -> Result> + Send>>, Error> { // TODO must apply the proper x-binning depending on the requested AggKind. - info!("make_event_pipe {:?}", evq); let evq = evq.clone(); let DirAndPrefix { dir, prefix } = directory_for_channel_files(&evq.channel, base_path)?; diff --git a/archapp/src/eventsitem.rs b/archapp/src/eventsitem.rs new file mode 100644 index 0000000..f7d1aa0 --- /dev/null +++ b/archapp/src/eventsitem.rs @@ -0,0 +1,156 @@ +use items::{Appendable, Clearable, PushableIndex, WithLen, WithTimestamps}; +use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape}; + +use crate::{ + binnedevents::XBinnedEvents, + plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents}, +}; + +#[derive(Debug)] +pub enum EventsItem { + Plain(PlainEvents), + XBinnedEvents(XBinnedEvents), +} + +impl EventsItem { + pub fn is_wave(&self) -> bool { + use EventsItem::*; + match self { + Plain(h) => h.is_wave(), + XBinnedEvents(h) => { + if let Shape::Wave(_) = h.shape() { + true + } else { + false + } + } + } + } + + pub fn variant_name(&self) -> String { + use EventsItem::*; + match self { + Plain(h) => format!("Plain({})", h.variant_name()), + XBinnedEvents(h) => format!("Plain({})", h.variant_name()), + } + } + + pub fn x_aggregate(self, ak: &AggKind) -> Self { + use EventsItem::*; + match self { + Plain(k) => k.x_aggregate(ak), + XBinnedEvents(k) => k.x_aggregate(ak), + } + } + + pub fn type_info(&self) -> (ScalarType, Shape) { + match self { + EventsItem::Plain(k) => match k { + PlainEvents::Scalar(k) => match k { + ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar), + ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar), + ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar), + ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar), + ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar), + }, + PlainEvents::Wave(k) => match k { + // TODO + // Inherent issue for the non-static-type backends: + // there is a chance that we can't determine the shape here. + WavePlainEvents::Byte(k) => (ScalarType::I8, k.shape().unwrap()), + WavePlainEvents::Short(k) => (ScalarType::I16, k.shape().unwrap()), + WavePlainEvents::Int(k) => (ScalarType::I32, k.shape().unwrap()), + WavePlainEvents::Float(k) => (ScalarType::F32, k.shape().unwrap()), + WavePlainEvents::Double(k) => (ScalarType::F64, k.shape().unwrap()), + }, + }, + EventsItem::XBinnedEvents(_k) => panic!(), + } + } +} + +impl WithLen for EventsItem { + fn len(&self) -> usize { + use EventsItem::*; + match self { + Plain(j) => j.len(), + XBinnedEvents(j) => j.len(), + } + } +} + +impl WithTimestamps for EventsItem { + fn ts(&self, ix: usize) -> u64 { + use EventsItem::*; + match self { + Plain(j) => j.ts(ix), + XBinnedEvents(j) => j.ts(ix), + } + } +} + +impl Appendable for EventsItem { + fn empty_like_self(&self) -> Self { + match self { + EventsItem::Plain(k) => EventsItem::Plain(k.empty_like_self()), + EventsItem::XBinnedEvents(k) => EventsItem::XBinnedEvents(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Plain(k) => match src { + Self::Plain(j) => k.append(j), + _ => panic!(), + }, + Self::XBinnedEvents(k) => match src { + Self::XBinnedEvents(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for EventsItem { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Plain(k) => match src { + Self::Plain(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::XBinnedEvents(k) => match src { + Self::XBinnedEvents(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl Clearable for EventsItem { + fn clear(&mut self) { + match self { + EventsItem::Plain(k) => k.clear(), + EventsItem::XBinnedEvents(k) => k.clear(), + } + } +} + +impl HasShape for EventsItem { + fn shape(&self) -> Shape { + use EventsItem::*; + match self { + Plain(h) => h.shape(), + XBinnedEvents(h) => h.shape(), + } + } +} + +impl HasScalarType for EventsItem { + fn scalar_type(&self) -> ScalarType { + use EventsItem::*; + match self { + Plain(h) => h.scalar_type(), + XBinnedEvents(h) => h.scalar_type(), + } + } +} diff --git a/archapp/src/lib.rs b/archapp/src/lib.rs index 4ede65a..2d62444 100644 --- a/archapp/src/lib.rs +++ b/archapp/src/lib.rs @@ -3,26 +3,28 @@ pub mod generated; #[cfg(not(feature = "devread"))] pub mod generated {} pub mod archeng; +pub mod binnedevents; +pub mod events; +pub mod eventsitem; #[cfg(feature = "devread")] pub mod parse; #[cfg(not(feature = "devread"))] pub mod parsestub; - -use err::Error; -use items::eventvalues::EventValues; -use items::numops::NumOps; -use items::waveevents::{WaveEvents, WaveXBinner}; -use items::xbinnedscalarevents::XBinnedScalarEvents; -use items::xbinnedwaveevents::XBinnedWaveEvents; -use items::{Appendable, Clearable, EventsNodeProcessor, PushableIndex, SitemtyFrameType, WithLen, WithTimestamps}; -use netpod::{AggKind, ByteOrder, HasScalarType, HasShape, ScalarType, Shape}; -#[cfg(not(feature = "devread"))] -pub use parsestub as parse; - -pub mod events; +pub mod plainevents; +pub mod storagemerge; #[cfg(feature = "devread")] #[cfg(test)] pub mod test; +pub mod timed; + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use async_channel::Sender; +use err::Error; +use futures_core::Future; +use netpod::log::*; +#[cfg(not(feature = "devread"))] +pub use parsestub as parse; fn unescape_archapp_msg(inp: &[u8], mut ret: Vec) -> Result, Error> { ret.clear(); @@ -48,1078 +50,30 @@ fn unescape_archapp_msg(inp: &[u8], mut ret: Vec) -> Result, Error> Ok(ret) } -#[derive(Debug)] -pub enum ScalarPlainEvents { - Byte(EventValues), - Short(EventValues), - Int(EventValues), - Float(EventValues), - Double(EventValues), -} +static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0); -impl ScalarPlainEvents { - pub fn variant_name(&self) -> String { - use ScalarPlainEvents::*; - match self { - Byte(_) => format!("Byte"), - Short(_) => format!("Short"), - Int(_) => format!("Int"), - Float(_) => format!("Float"), - Double(_) => format!("Double"), - } +fn channel_send_error() { + let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel); + if c < 10 { + error!("CHANNEL_SEND_ERROR {}", c); } } -impl Clearable for ScalarPlainEvents { - fn clear(&mut self) { - match self { - ScalarPlainEvents::Byte(k) => k.clear(), - ScalarPlainEvents::Short(k) => k.clear(), - ScalarPlainEvents::Int(k) => k.clear(), - ScalarPlainEvents::Float(k) => k.clear(), - ScalarPlainEvents::Double(k) => k.clear(), - } - } -} - -impl Appendable for ScalarPlainEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Byte(k) => Self::Byte(k.empty_like_self()), - Self::Short(k) => Self::Short(k.empty_like_self()), - Self::Int(k) => Self::Int(k.empty_like_self()), - Self::Float(k) => Self::Float(k.empty_like_self()), - Self::Double(k) => Self::Double(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.append(j), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.append(j), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.append(j), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.append(j), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for ScalarPlainEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for ScalarPlainEvents { - fn len(&self) -> usize { - use ScalarPlainEvents::*; - match self { - Byte(j) => j.len(), - Short(j) => j.len(), - Int(j) => j.len(), - Float(j) => j.len(), - Double(j) => j.len(), - } - } -} - -impl WithTimestamps for ScalarPlainEvents { - fn ts(&self, ix: usize) -> u64 { - use ScalarPlainEvents::*; - match self { - Byte(j) => j.ts(ix), - Short(j) => j.ts(ix), - Int(j) => j.ts(ix), - Float(j) => j.ts(ix), - Double(j) => j.ts(ix), - } - } -} - -impl HasShape for ScalarPlainEvents { - fn shape(&self) -> Shape { - match self { - _ => Shape::Scalar, - } - } -} - -impl HasScalarType for ScalarPlainEvents { - fn scalar_type(&self) -> ScalarType { - use ScalarPlainEvents::*; - match self { - Byte(_) => ScalarType::I8, - Short(_) => ScalarType::I16, - Int(_) => ScalarType::I32, - Float(_) => ScalarType::F32, - Double(_) => ScalarType::F64, - } - } -} - -#[derive(Debug)] -pub enum WavePlainEvents { - Byte(WaveEvents), - Short(WaveEvents), - Int(WaveEvents), - Float(WaveEvents), - Double(WaveEvents), -} - -impl WavePlainEvents { - pub fn shape(&self) -> Result { - match self { - WavePlainEvents::Byte(k) => k.shape(), - WavePlainEvents::Short(k) => k.shape(), - WavePlainEvents::Int(k) => k.shape(), - WavePlainEvents::Float(k) => k.shape(), - WavePlainEvents::Double(k) => k.shape(), - } - } -} - -fn _tmp1() { - let _ev = EventValues:: { - tss: vec![], - values: vec![], - }; - ::is_nan(err::todoval()); - if as SitemtyFrameType>::FRAME_TYPE_ID == 0 { - // Just a dummy.. - panic!(); - } - // as NumOps>::is_nan(err::todoval()); - //> as SitemtyFrameType>::FRAME_TYPE_ID; -} - -macro_rules! wagg1 { - ($k:expr, $ak:expr, $shape:expr, $sty:ident) => { - match $ak { - AggKind::EventBlobs => panic!(), - AggKind::Plain => EventsItem::Plain(PlainEvents::Wave(WavePlainEvents::$sty($k))), - AggKind::TimeWeightedScalar => { - let p = WaveXBinner::create($shape, $ak.clone()); - let j = p.process($k); - EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j))) - } - AggKind::DimXBins1 => { - let p = WaveXBinner::create($shape, $ak.clone()); - let j = p.process($k); - EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j))) - } - AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), - } - }; -} - -impl WavePlainEvents { - pub fn variant_name(&self) -> String { - use WavePlainEvents::*; - match self { - Byte(h) => format!("Byte({})", h.vals.first().map_or(0, |j| j.len())), - Short(h) => format!("Short({})", h.vals.first().map_or(0, |j| j.len())), - Int(h) => format!("Int({})", h.vals.first().map_or(0, |j| j.len())), - Float(h) => format!("Float({})", h.vals.first().map_or(0, |j| j.len())), - Double(h) => format!("Double({})", h.vals.first().map_or(0, |j| j.len())), - } - } - - fn x_aggregate(self, ak: &AggKind) -> EventsItem { - use WavePlainEvents::*; - let shape = self.shape().unwrap(); - match self { - Byte(k) => wagg1!(k, ak, shape, Byte), - Short(k) => wagg1!(k, ak, shape, Short), - Int(k) => wagg1!(k, ak, shape, Int), - Float(k) => wagg1!(k, ak, shape, Float), - Double(k) => wagg1!(k, ak, shape, Double), - } - } -} - -impl Clearable for WavePlainEvents { - fn clear(&mut self) { - match self { - WavePlainEvents::Byte(k) => k.clear(), - WavePlainEvents::Short(k) => k.clear(), - WavePlainEvents::Int(k) => k.clear(), - WavePlainEvents::Float(k) => k.clear(), - WavePlainEvents::Double(k) => k.clear(), - } - } -} - -impl Appendable for WavePlainEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Byte(k) => Self::Byte(k.empty_like_self()), - Self::Short(k) => Self::Short(k.empty_like_self()), - Self::Int(k) => Self::Int(k.empty_like_self()), - Self::Float(k) => Self::Float(k.empty_like_self()), - Self::Double(k) => Self::Double(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.append(j), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.append(j), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.append(j), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.append(j), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for WavePlainEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for WavePlainEvents { - fn len(&self) -> usize { - use WavePlainEvents::*; - match self { - Byte(j) => j.len(), - Short(j) => j.len(), - Int(j) => j.len(), - Float(j) => j.len(), - Double(j) => j.len(), - } - } -} - -impl WithTimestamps for WavePlainEvents { - fn ts(&self, ix: usize) -> u64 { - use WavePlainEvents::*; - match self { - Byte(j) => j.ts(ix), - Short(j) => j.ts(ix), - Int(j) => j.ts(ix), - Float(j) => j.ts(ix), - Double(j) => j.ts(ix), - } - } -} - -impl HasShape for WavePlainEvents { - fn shape(&self) -> Shape { - /*use WavePlainEvents::*; - match self { - Byte(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), - Short(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), - Int(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), - Float(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), - Double(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), - }*/ - self.shape().unwrap() - } -} - -impl HasScalarType for WavePlainEvents { - fn scalar_type(&self) -> ScalarType { - use WavePlainEvents::*; - match self { - Byte(_) => ScalarType::I8, - Short(_) => ScalarType::I16, - Int(_) => ScalarType::I32, - Float(_) => ScalarType::F32, - Double(_) => ScalarType::F64, - } - } -} - -#[derive(Debug)] -pub enum MultiBinWaveEvents { - Byte(XBinnedWaveEvents), - Short(XBinnedWaveEvents), - Int(XBinnedWaveEvents), - Float(XBinnedWaveEvents), - Double(XBinnedWaveEvents), -} - -impl MultiBinWaveEvents { - pub fn variant_name(&self) -> String { - use MultiBinWaveEvents::*; - match self { - Byte(_) => format!("Byte"), - Short(_) => format!("Short"), - Int(_) => format!("Int"), - Float(_) => format!("Float"), - Double(_) => format!("Double"), - } - } - - fn x_aggregate(self, ak: &AggKind) -> EventsItem { - use MultiBinWaveEvents::*; - match self { - Byte(k) => match ak { - AggKind::EventBlobs => panic!(), - AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::MultiBinWave(MultiBinWaveEvents::Byte(k))), - AggKind::TimeWeightedScalar => err::todoval(), - AggKind::DimXBins1 => err::todoval(), - AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), - }, - _ => err::todoval(), - } - } -} - -impl Clearable for MultiBinWaveEvents { - fn clear(&mut self) { - match self { - MultiBinWaveEvents::Byte(k) => k.clear(), - MultiBinWaveEvents::Short(k) => k.clear(), - MultiBinWaveEvents::Int(k) => k.clear(), - MultiBinWaveEvents::Float(k) => k.clear(), - MultiBinWaveEvents::Double(k) => k.clear(), - } - } -} - -impl Appendable for MultiBinWaveEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Byte(k) => Self::Byte(k.empty_like_self()), - Self::Short(k) => Self::Short(k.empty_like_self()), - Self::Int(k) => Self::Int(k.empty_like_self()), - Self::Float(k) => Self::Float(k.empty_like_self()), - Self::Double(k) => Self::Double(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.append(j), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.append(j), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.append(j), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.append(j), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for MultiBinWaveEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for MultiBinWaveEvents { - fn len(&self) -> usize { - use MultiBinWaveEvents::*; - match self { - Byte(j) => j.len(), - Short(j) => j.len(), - Int(j) => j.len(), - Float(j) => j.len(), - Double(j) => j.len(), - } - } -} - -impl WithTimestamps for MultiBinWaveEvents { - fn ts(&self, ix: usize) -> u64 { - use MultiBinWaveEvents::*; - match self { - Byte(j) => j.ts(ix), - Short(j) => j.ts(ix), - Int(j) => j.ts(ix), - Float(j) => j.ts(ix), - Double(j) => j.ts(ix), - } - } -} - -impl HasShape for MultiBinWaveEvents { - fn shape(&self) -> Shape { - use MultiBinWaveEvents::*; - match self { - Byte(_) => Shape::Scalar, - Short(_) => Shape::Scalar, - Int(_) => Shape::Scalar, - Float(_) => Shape::Scalar, - Double(_) => Shape::Scalar, - } - } -} - -impl HasScalarType for MultiBinWaveEvents { - fn scalar_type(&self) -> ScalarType { - use MultiBinWaveEvents::*; - match self { - Byte(_) => ScalarType::I8, - Short(_) => ScalarType::I16, - Int(_) => ScalarType::I32, - Float(_) => ScalarType::F32, - Double(_) => ScalarType::F64, - } - } -} - -#[derive(Debug)] -pub enum SingleBinWaveEvents { - Byte(XBinnedScalarEvents), - Short(XBinnedScalarEvents), - Int(XBinnedScalarEvents), - Float(XBinnedScalarEvents), - Double(XBinnedScalarEvents), -} - -impl SingleBinWaveEvents { - pub fn variant_name(&self) -> String { - use SingleBinWaveEvents::*; - match self { - Byte(_) => format!("Byte"), - Short(_) => format!("Short"), - Int(_) => format!("Int"), - Float(_) => format!("Float"), - Double(_) => format!("Double"), - } - } - - fn x_aggregate(self, ak: &AggKind) -> EventsItem { - use SingleBinWaveEvents::*; - match self { - Byte(k) => match ak { - AggKind::EventBlobs => panic!(), - AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::Byte(k))), - AggKind::TimeWeightedScalar => err::todoval(), - AggKind::DimXBins1 => err::todoval(), - AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), - }, - _ => err::todoval(), - } - } -} - -impl Clearable for SingleBinWaveEvents { - fn clear(&mut self) { - match self { - SingleBinWaveEvents::Byte(k) => k.clear(), - SingleBinWaveEvents::Short(k) => k.clear(), - SingleBinWaveEvents::Int(k) => k.clear(), - SingleBinWaveEvents::Float(k) => k.clear(), - SingleBinWaveEvents::Double(k) => k.clear(), - } - } -} - -impl Appendable for SingleBinWaveEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Byte(k) => Self::Byte(k.empty_like_self()), - Self::Short(k) => Self::Short(k.empty_like_self()), - Self::Int(k) => Self::Int(k.empty_like_self()), - Self::Float(k) => Self::Float(k.empty_like_self()), - Self::Double(k) => Self::Double(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.append(j), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.append(j), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.append(j), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.append(j), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for SingleBinWaveEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Byte(k) => match src { - Self::Byte(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Short(k) => match src { - Self::Short(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Int(k) => match src { - Self::Int(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Float(k) => match src { - Self::Float(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Double(k) => match src { - Self::Double(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for SingleBinWaveEvents { - fn len(&self) -> usize { - use SingleBinWaveEvents::*; - match self { - Byte(j) => j.len(), - Short(j) => j.len(), - Int(j) => j.len(), - Float(j) => j.len(), - Double(j) => j.len(), - } - } -} - -impl WithTimestamps for SingleBinWaveEvents { - fn ts(&self, ix: usize) -> u64 { - use SingleBinWaveEvents::*; - match self { - Byte(j) => j.ts(ix), - Short(j) => j.ts(ix), - Int(j) => j.ts(ix), - Float(j) => j.ts(ix), - Double(j) => j.ts(ix), - } - } -} - -impl HasShape for SingleBinWaveEvents { - fn shape(&self) -> Shape { - use SingleBinWaveEvents::*; - match self { - Byte(_) => Shape::Scalar, - Short(_) => Shape::Scalar, - Int(_) => Shape::Scalar, - Float(_) => Shape::Scalar, - Double(_) => Shape::Scalar, - } - } -} - -impl HasScalarType for SingleBinWaveEvents { - fn scalar_type(&self) -> ScalarType { - use SingleBinWaveEvents::*; - match self { - Byte(_) => ScalarType::I8, - Short(_) => ScalarType::I16, - Int(_) => ScalarType::I32, - Float(_) => ScalarType::F32, - Double(_) => ScalarType::F64, - } - } -} - -#[derive(Debug)] -pub enum XBinnedEvents { - Scalar(ScalarPlainEvents), - SingleBinWave(SingleBinWaveEvents), - MultiBinWave(MultiBinWaveEvents), -} - -impl XBinnedEvents { - pub fn variant_name(&self) -> String { - use XBinnedEvents::*; - match self { - Scalar(h) => format!("Scalar({})", h.variant_name()), - SingleBinWave(h) => format!("SingleBinWave({})", h.variant_name()), - MultiBinWave(h) => format!("MultiBinWave({})", h.variant_name()), - } - } - - pub fn x_aggregate(self, ak: &AggKind) -> EventsItem { - use XBinnedEvents::*; - match self { - Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)), - SingleBinWave(k) => k.x_aggregate(ak), - MultiBinWave(k) => k.x_aggregate(ak), - } - } -} - -impl Clearable for XBinnedEvents { - fn clear(&mut self) { - match self { - XBinnedEvents::Scalar(k) => k.clear(), - XBinnedEvents::SingleBinWave(k) => k.clear(), - XBinnedEvents::MultiBinWave(k) => k.clear(), - } - } -} - -impl Appendable for XBinnedEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Scalar(k) => Self::Scalar(k.empty_like_self()), - Self::SingleBinWave(k) => Self::SingleBinWave(k.empty_like_self()), - Self::MultiBinWave(k) => Self::MultiBinWave(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Scalar(k) => match src { - Self::Scalar(j) => k.append(j), - _ => panic!(), - }, - Self::SingleBinWave(k) => match src { - Self::SingleBinWave(j) => k.append(j), - _ => panic!(), - }, - Self::MultiBinWave(k) => match src { - Self::MultiBinWave(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for XBinnedEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Scalar(k) => match src { - Self::Scalar(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::SingleBinWave(k) => match src { - Self::SingleBinWave(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::MultiBinWave(k) => match src { - Self::MultiBinWave(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for XBinnedEvents { - fn len(&self) -> usize { - use XBinnedEvents::*; - match self { - Scalar(j) => j.len(), - SingleBinWave(j) => j.len(), - MultiBinWave(j) => j.len(), - } - } -} - -impl WithTimestamps for XBinnedEvents { - fn ts(&self, ix: usize) -> u64 { - use XBinnedEvents::*; - match self { - Scalar(j) => j.ts(ix), - SingleBinWave(j) => j.ts(ix), - MultiBinWave(j) => j.ts(ix), - } - } -} - -impl HasShape for XBinnedEvents { - fn shape(&self) -> Shape { - use XBinnedEvents::*; - match self { - Scalar(h) => h.shape(), - SingleBinWave(h) => h.shape(), - MultiBinWave(h) => h.shape(), - } - } -} - -impl HasScalarType for XBinnedEvents { - fn scalar_type(&self) -> ScalarType { - use XBinnedEvents::*; - match self { - Scalar(h) => h.scalar_type(), - SingleBinWave(h) => h.scalar_type(), - MultiBinWave(h) => h.scalar_type(), - } - } -} - -#[derive(Debug)] -pub enum PlainEvents { - Scalar(ScalarPlainEvents), - Wave(WavePlainEvents), -} - -impl PlainEvents { - pub fn is_wave(&self) -> bool { - use PlainEvents::*; - match self { - Scalar(_) => false, - Wave(_) => true, - } - } - - pub fn variant_name(&self) -> String { - use PlainEvents::*; - match self { - Scalar(h) => format!("Scalar({})", h.variant_name()), - Wave(h) => format!("Scalar({})", h.variant_name()), - } - } - - pub fn x_aggregate(self, ak: &AggKind) -> EventsItem { - use PlainEvents::*; - match self { - Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)), - Wave(k) => k.x_aggregate(ak), - } - } -} - -impl Clearable for PlainEvents { - fn clear(&mut self) { - match self { - PlainEvents::Scalar(k) => k.clear(), - PlainEvents::Wave(k) => k.clear(), - } - } -} - -impl Appendable for PlainEvents { - fn empty_like_self(&self) -> Self { - match self { - Self::Scalar(k) => Self::Scalar(k.empty_like_self()), - Self::Wave(k) => Self::Wave(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - PlainEvents::Scalar(k) => match src { - Self::Scalar(j) => k.append(j), - _ => panic!(), - }, - PlainEvents::Wave(k) => match src { - Self::Wave(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for PlainEvents { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Scalar(k) => match src { - Self::Scalar(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::Wave(k) => match src { - Self::Wave(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl WithLen for PlainEvents { - fn len(&self) -> usize { - use PlainEvents::*; - match self { - Scalar(j) => j.len(), - Wave(j) => j.len(), - } - } -} - -impl WithTimestamps for PlainEvents { - fn ts(&self, ix: usize) -> u64 { - use PlainEvents::*; - match self { - Scalar(j) => j.ts(ix), - Wave(j) => j.ts(ix), - } - } -} - -impl HasShape for PlainEvents { - fn shape(&self) -> Shape { - use PlainEvents::*; - match self { - Scalar(h) => HasShape::shape(h), - Wave(h) => HasShape::shape(h), - } - } -} - -impl HasScalarType for PlainEvents { - fn scalar_type(&self) -> ScalarType { - use PlainEvents::*; - match self { - Scalar(h) => h.scalar_type(), - Wave(h) => h.scalar_type(), - } - } -} - -#[derive(Debug)] -pub enum EventsItem { - Plain(PlainEvents), - XBinnedEvents(XBinnedEvents), -} - -impl EventsItem { - pub fn is_wave(&self) -> bool { - use EventsItem::*; - match self { - Plain(h) => h.is_wave(), - XBinnedEvents(h) => { - if let Shape::Wave(_) = h.shape() { - true - } else { - false +fn wrap_task(task: T, tx: Sender>) +where + T: Future> + Send + 'static, + O1: Send + 'static, + O2: Send + 'static, +{ + let task = async move { + match task.await { + Ok(_) => {} + Err(e) => { + if let Err(_) = tx.send(Err(e)).await { + channel_send_error(); } } } - } - - pub fn variant_name(&self) -> String { - use EventsItem::*; - match self { - Plain(h) => format!("Plain({})", h.variant_name()), - XBinnedEvents(h) => format!("Plain({})", h.variant_name()), - } - } - - pub fn x_aggregate(self, ak: &AggKind) -> Self { - use EventsItem::*; - match self { - Plain(k) => k.x_aggregate(ak), - XBinnedEvents(k) => k.x_aggregate(ak), - } - } - - pub fn type_info(&self) -> (ScalarType, Shape) { - match self { - EventsItem::Plain(k) => match k { - PlainEvents::Scalar(k) => match k { - ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar), - ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar), - ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar), - ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar), - ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar), - }, - PlainEvents::Wave(k) => match k { - // TODO - // Inherent issue for the non-static-type backends: - // there is a chance that we can't determine the shape here. - WavePlainEvents::Byte(k) => (ScalarType::I8, k.shape().unwrap()), - WavePlainEvents::Short(k) => (ScalarType::I16, k.shape().unwrap()), - WavePlainEvents::Int(k) => (ScalarType::I32, k.shape().unwrap()), - WavePlainEvents::Float(k) => (ScalarType::F32, k.shape().unwrap()), - WavePlainEvents::Double(k) => (ScalarType::F64, k.shape().unwrap()), - }, - }, - EventsItem::XBinnedEvents(k) => panic!(), - } - } -} - -impl WithLen for EventsItem { - fn len(&self) -> usize { - use EventsItem::*; - match self { - Plain(j) => j.len(), - XBinnedEvents(j) => j.len(), - } - } -} - -impl WithTimestamps for EventsItem { - fn ts(&self, ix: usize) -> u64 { - use EventsItem::*; - match self { - Plain(j) => j.ts(ix), - XBinnedEvents(j) => j.ts(ix), - } - } -} - -impl Appendable for EventsItem { - fn empty_like_self(&self) -> Self { - match self { - EventsItem::Plain(k) => EventsItem::Plain(k.empty_like_self()), - EventsItem::XBinnedEvents(k) => EventsItem::XBinnedEvents(k.empty_like_self()), - } - } - - fn append(&mut self, src: &Self) { - match self { - Self::Plain(k) => match src { - Self::Plain(j) => k.append(j), - _ => panic!(), - }, - Self::XBinnedEvents(k) => match src { - Self::XBinnedEvents(j) => k.append(j), - _ => panic!(), - }, - } - } -} - -impl PushableIndex for EventsItem { - fn push_index(&mut self, src: &Self, ix: usize) { - match self { - Self::Plain(k) => match src { - Self::Plain(j) => k.push_index(j, ix), - _ => panic!(), - }, - Self::XBinnedEvents(k) => match src { - Self::XBinnedEvents(j) => k.push_index(j, ix), - _ => panic!(), - }, - } - } -} - -impl Clearable for EventsItem { - fn clear(&mut self) { - match self { - EventsItem::Plain(k) => k.clear(), - EventsItem::XBinnedEvents(k) => k.clear(), - } - } -} - -impl HasShape for EventsItem { - fn shape(&self) -> Shape { - use EventsItem::*; - match self { - Plain(h) => h.shape(), - XBinnedEvents(h) => h.shape(), - } - } -} - -impl HasScalarType for EventsItem { - fn scalar_type(&self) -> ScalarType { - use EventsItem::*; - match self { - Plain(h) => h.scalar_type(), - XBinnedEvents(h) => h.scalar_type(), - } - } + }; + taskrun::spawn(task); } diff --git a/archapp/src/parse.rs b/archapp/src/parse.rs index 6e36035..4b4eef2 100644 --- a/archapp/src/parse.rs +++ b/archapp/src/parse.rs @@ -1,8 +1,10 @@ pub mod multi; use crate::events::parse_data_filename; +use crate::eventsitem::EventsItem; use crate::generated::EPICSEvent::PayloadType; -use crate::{unescape_archapp_msg, EventsItem, PlainEvents, ScalarPlainEvents, WavePlainEvents}; +use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents}; +use crate::unescape_archapp_msg; use archapp_xc::*; use async_channel::{bounded, Receiver}; use chrono::{TimeZone, Utc}; diff --git a/archapp/src/plainevents.rs b/archapp/src/plainevents.rs new file mode 100644 index 0000000..34e6605 --- /dev/null +++ b/archapp/src/plainevents.rs @@ -0,0 +1,465 @@ +use crate::binnedevents::{SingleBinWaveEvents, XBinnedEvents}; +use crate::eventsitem::EventsItem; +use err::Error; +use items::eventvalues::EventValues; +use items::waveevents::{WaveEvents, WaveXBinner}; +use items::{Appendable, Clearable, EventsNodeProcessor, PushableIndex, WithLen, WithTimestamps}; +use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape}; + +#[derive(Debug)] +pub enum ScalarPlainEvents { + Byte(EventValues), + Short(EventValues), + Int(EventValues), + Float(EventValues), + Double(EventValues), +} + +impl ScalarPlainEvents { + pub fn variant_name(&self) -> String { + use ScalarPlainEvents::*; + match self { + Byte(_) => format!("Byte"), + Short(_) => format!("Short"), + Int(_) => format!("Int"), + Float(_) => format!("Float"), + Double(_) => format!("Double"), + } + } +} + +impl Clearable for ScalarPlainEvents { + fn clear(&mut self) { + match self { + ScalarPlainEvents::Byte(k) => k.clear(), + ScalarPlainEvents::Short(k) => k.clear(), + ScalarPlainEvents::Int(k) => k.clear(), + ScalarPlainEvents::Float(k) => k.clear(), + ScalarPlainEvents::Double(k) => k.clear(), + } + } +} + +impl Appendable for ScalarPlainEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Byte(k) => Self::Byte(k.empty_like_self()), + Self::Short(k) => Self::Short(k.empty_like_self()), + Self::Int(k) => Self::Int(k.empty_like_self()), + Self::Float(k) => Self::Float(k.empty_like_self()), + Self::Double(k) => Self::Double(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.append(j), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.append(j), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.append(j), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.append(j), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for ScalarPlainEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for ScalarPlainEvents { + fn len(&self) -> usize { + use ScalarPlainEvents::*; + match self { + Byte(j) => j.len(), + Short(j) => j.len(), + Int(j) => j.len(), + Float(j) => j.len(), + Double(j) => j.len(), + } + } +} + +impl WithTimestamps for ScalarPlainEvents { + fn ts(&self, ix: usize) -> u64 { + use ScalarPlainEvents::*; + match self { + Byte(j) => j.ts(ix), + Short(j) => j.ts(ix), + Int(j) => j.ts(ix), + Float(j) => j.ts(ix), + Double(j) => j.ts(ix), + } + } +} + +impl HasShape for ScalarPlainEvents { + fn shape(&self) -> Shape { + match self { + _ => Shape::Scalar, + } + } +} + +impl HasScalarType for ScalarPlainEvents { + fn scalar_type(&self) -> ScalarType { + use ScalarPlainEvents::*; + match self { + Byte(_) => ScalarType::I8, + Short(_) => ScalarType::I16, + Int(_) => ScalarType::I32, + Float(_) => ScalarType::F32, + Double(_) => ScalarType::F64, + } + } +} + +#[derive(Debug)] +pub enum WavePlainEvents { + Byte(WaveEvents), + Short(WaveEvents), + Int(WaveEvents), + Float(WaveEvents), + Double(WaveEvents), +} + +impl WavePlainEvents { + pub fn shape(&self) -> Result { + match self { + WavePlainEvents::Byte(k) => k.shape(), + WavePlainEvents::Short(k) => k.shape(), + WavePlainEvents::Int(k) => k.shape(), + WavePlainEvents::Float(k) => k.shape(), + WavePlainEvents::Double(k) => k.shape(), + } + } +} + +macro_rules! wagg1 { + ($k:expr, $ak:expr, $shape:expr, $sty:ident) => { + match $ak { + AggKind::EventBlobs => panic!(), + AggKind::Plain => EventsItem::Plain(PlainEvents::Wave(WavePlainEvents::$sty($k))), + AggKind::TimeWeightedScalar => { + let p = WaveXBinner::create($shape, $ak.clone()); + let j = p.process($k); + EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j))) + } + AggKind::DimXBins1 => { + let p = WaveXBinner::create($shape, $ak.clone()); + let j = p.process($k); + EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j))) + } + AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())), + } + }; +} + +impl WavePlainEvents { + pub fn variant_name(&self) -> String { + use WavePlainEvents::*; + match self { + Byte(h) => format!("Byte({})", h.vals.first().map_or(0, |j| j.len())), + Short(h) => format!("Short({})", h.vals.first().map_or(0, |j| j.len())), + Int(h) => format!("Int({})", h.vals.first().map_or(0, |j| j.len())), + Float(h) => format!("Float({})", h.vals.first().map_or(0, |j| j.len())), + Double(h) => format!("Double({})", h.vals.first().map_or(0, |j| j.len())), + } + } + + fn x_aggregate(self, ak: &AggKind) -> EventsItem { + use WavePlainEvents::*; + let shape = self.shape().unwrap(); + match self { + Byte(k) => wagg1!(k, ak, shape, Byte), + Short(k) => wagg1!(k, ak, shape, Short), + Int(k) => wagg1!(k, ak, shape, Int), + Float(k) => wagg1!(k, ak, shape, Float), + Double(k) => wagg1!(k, ak, shape, Double), + } + } +} + +impl Clearable for WavePlainEvents { + fn clear(&mut self) { + match self { + WavePlainEvents::Byte(k) => k.clear(), + WavePlainEvents::Short(k) => k.clear(), + WavePlainEvents::Int(k) => k.clear(), + WavePlainEvents::Float(k) => k.clear(), + WavePlainEvents::Double(k) => k.clear(), + } + } +} + +impl Appendable for WavePlainEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Byte(k) => Self::Byte(k.empty_like_self()), + Self::Short(k) => Self::Short(k.empty_like_self()), + Self::Int(k) => Self::Int(k.empty_like_self()), + Self::Float(k) => Self::Float(k.empty_like_self()), + Self::Double(k) => Self::Double(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.append(j), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.append(j), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.append(j), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.append(j), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for WavePlainEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Byte(k) => match src { + Self::Byte(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Short(k) => match src { + Self::Short(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Int(k) => match src { + Self::Int(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Float(k) => match src { + Self::Float(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Double(k) => match src { + Self::Double(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for WavePlainEvents { + fn len(&self) -> usize { + use WavePlainEvents::*; + match self { + Byte(j) => j.len(), + Short(j) => j.len(), + Int(j) => j.len(), + Float(j) => j.len(), + Double(j) => j.len(), + } + } +} + +impl WithTimestamps for WavePlainEvents { + fn ts(&self, ix: usize) -> u64 { + use WavePlainEvents::*; + match self { + Byte(j) => j.ts(ix), + Short(j) => j.ts(ix), + Int(j) => j.ts(ix), + Float(j) => j.ts(ix), + Double(j) => j.ts(ix), + } + } +} + +impl HasShape for WavePlainEvents { + fn shape(&self) -> Shape { + /*use WavePlainEvents::*; + match self { + Byte(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), + Short(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), + Int(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), + Float(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), + Double(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)), + }*/ + self.shape().unwrap() + } +} + +impl HasScalarType for WavePlainEvents { + fn scalar_type(&self) -> ScalarType { + use WavePlainEvents::*; + match self { + Byte(_) => ScalarType::I8, + Short(_) => ScalarType::I16, + Int(_) => ScalarType::I32, + Float(_) => ScalarType::F32, + Double(_) => ScalarType::F64, + } + } +} + +#[derive(Debug)] +pub enum PlainEvents { + Scalar(ScalarPlainEvents), + Wave(WavePlainEvents), +} + +impl PlainEvents { + pub fn is_wave(&self) -> bool { + use PlainEvents::*; + match self { + Scalar(_) => false, + Wave(_) => true, + } + } + + pub fn variant_name(&self) -> String { + use PlainEvents::*; + match self { + Scalar(h) => format!("Scalar({})", h.variant_name()), + Wave(h) => format!("Scalar({})", h.variant_name()), + } + } + + pub fn x_aggregate(self, ak: &AggKind) -> EventsItem { + use PlainEvents::*; + match self { + Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)), + Wave(k) => k.x_aggregate(ak), + } + } +} + +impl Clearable for PlainEvents { + fn clear(&mut self) { + match self { + PlainEvents::Scalar(k) => k.clear(), + PlainEvents::Wave(k) => k.clear(), + } + } +} + +impl Appendable for PlainEvents { + fn empty_like_self(&self) -> Self { + match self { + Self::Scalar(k) => Self::Scalar(k.empty_like_self()), + Self::Wave(k) => Self::Wave(k.empty_like_self()), + } + } + + fn append(&mut self, src: &Self) { + match self { + PlainEvents::Scalar(k) => match src { + Self::Scalar(j) => k.append(j), + _ => panic!(), + }, + PlainEvents::Wave(k) => match src { + Self::Wave(j) => k.append(j), + _ => panic!(), + }, + } + } +} + +impl PushableIndex for PlainEvents { + fn push_index(&mut self, src: &Self, ix: usize) { + match self { + Self::Scalar(k) => match src { + Self::Scalar(j) => k.push_index(j, ix), + _ => panic!(), + }, + Self::Wave(k) => match src { + Self::Wave(j) => k.push_index(j, ix), + _ => panic!(), + }, + } + } +} + +impl WithLen for PlainEvents { + fn len(&self) -> usize { + use PlainEvents::*; + match self { + Scalar(j) => j.len(), + Wave(j) => j.len(), + } + } +} + +impl WithTimestamps for PlainEvents { + fn ts(&self, ix: usize) -> u64 { + use PlainEvents::*; + match self { + Scalar(j) => j.ts(ix), + Wave(j) => j.ts(ix), + } + } +} + +impl HasShape for PlainEvents { + fn shape(&self) -> Shape { + use PlainEvents::*; + match self { + Scalar(h) => HasShape::shape(h), + Wave(h) => HasShape::shape(h), + } + } +} + +impl HasScalarType for PlainEvents { + fn scalar_type(&self) -> ScalarType { + use PlainEvents::*; + match self { + Scalar(h) => h.scalar_type(), + Wave(h) => h.scalar_type(), + } + } +} diff --git a/archapp/src/storagemerge.rs b/archapp/src/storagemerge.rs new file mode 100644 index 0000000..27eb8ba --- /dev/null +++ b/archapp/src/storagemerge.rs @@ -0,0 +1,310 @@ +use err::Error; +use futures_core::Stream; +use futures_util::StreamExt; +use items::{ + inspect_timestamps, Appendable, LogItem, PushableIndex, RangeCompletableItem, Sitemty, StatsItem, StreamItem, +}; +use netpod::log::*; +use netpod::{NanoRange, Nanos}; +use std::collections::VecDeque; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::eventsitem::EventsItem; + +/** +Priority-Merge events from different candidate sources. + +Backends like Channel Archiver store the compacted "medium/long-term" data of a channel +in logically unrelated locations on disk with unspecified semantics and without a +common index over "short+medium+long term" data. +In order to deliver data even over the edge of such (possibly overlapping) datasources +without common look tables, the best we can do is fetch data from all sources and +combine them. StorageMerge is doing this combination. +*/ +pub struct StorageMerge { + inps: Vec> + Send>>>, + names: Vec, + range: NanoRange, + completed_inps: Vec, + range_complete: Vec, + current_inp_item: Vec>, + error_items: VecDeque, + log_items: VecDeque, + stats_items: VecDeque, + ourname: String, + inprng: usize, + data_done: bool, + done: bool, + complete: bool, +} + +impl StorageMerge { + pub fn new( + inps: Vec> + Send>>>, + names: Vec, + range: NanoRange, + ) -> Self { + assert_eq!(inps.len(), names.len()); + let n = inps.len(); + let mut h = crc32fast::Hasher::new(); + h.update( + &SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .subsec_nanos() + .to_le_bytes(), + ); + let ourname = format!("{:08x}", h.finalize()); + for (i, n) in names.iter().enumerate() { + debug!("[{}] {} {}", ourname, i, n); + } + Self { + inps, + names, + range, + completed_inps: vec![false; n], + range_complete: vec![false; n], + current_inp_item: (0..n).into_iter().map(|_| None).collect(), + error_items: VecDeque::new(), + log_items: VecDeque::new(), + stats_items: VecDeque::new(), + inprng: n - 1, + data_done: false, + done: false, + complete: false, + ourname, + } + } + + fn refill_if_needed(self: &mut Pin<&mut Self>, cx: &mut Context) -> Result { + use Poll::*; + let mut is_pending = false; + for i in 0..self.inps.len() { + while self.current_inp_item[i].is_none() && self.completed_inps[i] == false { + match self.inps[i].poll_next_unpin(cx) { + Ready(j) => match j { + Some(j) => match j { + Ok(j) => match j { + StreamItem::DataItem(j) => match j { + RangeCompletableItem::Data(j) => { + self.current_inp_item[i] = Some(j); + } + RangeCompletableItem::RangeComplete => { + self.range_complete[i] = true; + } + }, + StreamItem::Log(k) => { + self.log_items.push_back(k); + } + StreamItem::Stats(k) => { + self.stats_items.push_back(k); + } + }, + Err(e) => { + error!("inp err input {} {:?}", i, e); + self.error_items.push_back(e); + } + }, + None => { + self.completed_inps[i] = true; + } + }, + Pending => { + is_pending = true; + break; + } + } + } + } + Ok(is_pending) + } + + fn decide_next_item(&mut self) -> Result>, Error> { + let not_found = 99999; + let mut i1 = self.inprng; + let mut j1 = not_found; + let mut tsmin = u64::MAX; + let mut tsend = u64::MAX; + #[allow(unused)] + use items::{WithLen, WithTimestamps}; + loop { + if self.completed_inps[i1] { + } else { + match self.current_inp_item[i1].as_ref() { + None => panic!(), + Some(j) => { + if j.len() == 0 { + j1 = i1; + break; + } else { + let ts1 = j.ts(0); + let ts2 = j.ts(j.len() - 1); + if ts1 == u64::MAX || ts2 == u64::MAX { + panic!(); + } + trace!("[{}] consider {} {:?}", self.ourname, i1, Nanos::from_ns(ts1)); + if ts1 <= tsmin { + tsmin = ts1; + tsend = ts2; + j1 = i1; + trace!( + "[{}] switch to source {} / {} {}", + self.ourname, + i1, + self.inps.len(), + self.names[i1] + ); + self.inprng = i1; + } else { + } + } + } + } + } + if i1 == 0 { + break; + } + i1 -= 1; + } + let i1 = (); + let _ = i1; + if j1 >= not_found { + Ok(None) + } else { + trace!("[{}] decide for source {}", self.ourname, j1); + trace!("[{}] decided tsmin {:?}", self.ourname, Nanos::from_ns(tsmin)); + trace!("[{}] decided tsend {:?}", self.ourname, Nanos::from_ns(tsend)); + let mut j5 = not_found; + let mut tsmin2 = u64::MAX; + if self.inprng > 0 { + trace!("[{}] locate the next earliest timestamp", self.ourname); + let mut i5 = self.inprng - 1; + loop { + if self.completed_inps[i5] { + } else { + let j = self.current_inp_item[i5].as_ref().unwrap(); + if j.len() != 0 { + let ts1 = j.ts(0); + if ts1 == u64::MAX { + panic!(); + } + trace!( + "[{}] consider {} {:?} for next earliest", + self.ourname, + i5, + Nanos::from_ns(ts1) + ); + if ts1 <= tsmin2 { + tsmin2 = ts1; + j5 = i5; + } + } + } + if i5 == 0 { + break; + } + i5 -= 1; + } + } + trace!( + "[{}] decided tsmin2 {:?} next earliest timestamp source {}", + self.ourname, + Nanos::from_ns(tsmin2), + j5 + ); + let item = self.current_inp_item[j1].take().unwrap(); + let item = if j5 != not_found && tsmin2 != u64::MAX { + if tsend >= tsmin2 { + { + let tsmin = Nanos::from_ns(tsmin); + let tsend = Nanos::from_ns(tsend); + let tsmin2 = Nanos::from_ns(tsmin2); + trace!( + "[{}] NEED TO TRUNCATE THE BLOCK tsmin {:?} tsend {:?} tsmin2 {:?}", + self.ourname, + tsmin, + tsend, + tsmin2 + ); + } + let mut out = item.empty_like_self(); + for i in 0..item.len() { + let ts = item.ts(i); + if ts < tsmin2 { + out.push_index(&item, i); + } + } + out + } else { + item + } + } else { + item + }; + trace!("[{}] emit {} events", self.ourname, item.len()); + if false { + let s = inspect_timestamps(&item, self.range.clone()); + trace!("[{}] timestamps:\n{}", self.ourname, s); + } + Ok(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(item))))) + } + } +} + +impl Stream for StorageMerge { + type Item = Sitemty; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + use Poll::*; + loop { + break if self.complete { + panic!() + } else if self.done { + self.complete = true; + Ready(None) + } else if let Some(k) = self.error_items.pop_front() { + Ready(Some(Err(k))) + } else if let Some(k) = self.log_items.pop_front() { + Ready(Some(Ok(StreamItem::Log(k)))) + } else if let Some(k) = self.stats_items.pop_front() { + Ready(Some(Ok(StreamItem::Stats(k)))) + } else if self.data_done { + self.done = true; + continue; + } else { + match self.refill_if_needed(cx) { + Ok(is_pending) => { + if is_pending { + if self.log_items.len() == 0 && self.stats_items.len() == 0 { + Pending + } else { + continue; + } + } else if self.error_items.len() != 0 { + continue; + } else { + match self.decide_next_item() { + Ok(Some(j)) => Ready(Some(j)), + Ok(None) => { + self.data_done = true; + continue; + } + Err(e) => { + error!("impl Stream for StorageMerge {:?}", e); + Ready(Some(Err(e))) + } + } + } + } + Err(e) => { + error!("{}", e); + self.done = true; + Ready(Some(Err(e))) + } + } + }; + } + } +} diff --git a/archapp/src/timed.rs b/archapp/src/timed.rs new file mode 100644 index 0000000..8a4d974 --- /dev/null +++ b/archapp/src/timed.rs @@ -0,0 +1,28 @@ +use std::time::Instant; + +use netpod::log::*; + +pub struct Timed { + name: String, + ts1: Instant, +} + +impl Timed { + pub fn new(name: T) -> Self + where + T: ToString, + { + Self { + name: name.to_string(), + ts1: Instant::now(), + } + } +} + +impl Drop for Timed { + fn drop(&mut self) { + let ts2 = Instant::now(); + let dt = ts2.duration_since(self.ts1); + info!("Timed {} {:?}", self.name, dt); + } +} diff --git a/dbconn/Cargo.toml b/dbconn/Cargo.toml index 9f8c9dd..c378a71 100644 --- a/dbconn/Cargo.toml +++ b/dbconn/Cargo.toml @@ -18,7 +18,7 @@ bytes = "1.0.1" pin-project = "1.0.7" #async-channel = "1" #dashmap = "3" -tokio-postgres = { version = "0.7.2", features = ["runtime", "with-chrono-0_4", "with-serde_json-1"] } +tokio-postgres = { version = "0.7.4", features = ["runtime", "with-chrono-0_4", "with-serde_json-1"] } async-channel = "1.6" chrono = "0.4" regex = "1.5.4" diff --git a/dbconn/src/scan.rs b/dbconn/src/scan.rs index 926d2f5..c265b9e 100644 --- a/dbconn/src/scan.rs +++ b/dbconn/src/scan.rs @@ -335,8 +335,6 @@ pub async fn update_db_with_channel_names( let tx2 = tx.clone(); let db_config = db_config.clone(); let block1 = async move { - //return Err(Error::with_msg("some test error1")); - //tx.send(Err(Error::with_msg("some test error2"))).await?; let dbc = crate::create_connection(&db_config).await?; let node_disk_ident = get_node_disk_ident(&node_config, &dbc).await?; let c1 = Arc::new(RwLock::new(0u32)); diff --git a/disk/src/channelexec.rs b/disk/src/channelexec.rs index ab272c0..a092da8 100644 --- a/disk/src/channelexec.rs +++ b/disk/src/channelexec.rs @@ -332,6 +332,7 @@ where let mut collector = ::new_collector(bin_count_exp); let mut i1 = 0; let mut stream = stream; + let mut total_duration = Duration::ZERO; loop { let item = if i1 == 0 { stream.next().await @@ -357,11 +358,24 @@ where info!("collect_plain_events_json log {:?}", item); } } - StreamItem::Stats(item) => { - if do_log { - info!("collect_plain_events_json stats {:?}", item); - } - } + StreamItem::Stats(item) => match item { + items::StatsItem::EventDataReadStats(_) => {} + items::StatsItem::RangeFilterStats(_) => {} + items::StatsItem::DiskStats(item) => match item { + netpod::DiskStats::OpenStats(k) => { + total_duration += k.duration; + } + netpod::DiskStats::SeekStats(k) => { + total_duration += k.duration; + } + netpod::DiskStats::ReadStats(k) => { + total_duration += k.duration; + } + netpod::DiskStats::ReadExactStats(k) => { + total_duration += k.duration; + } + }, + }, StreamItem::DataItem(item) => match item { RangeCompletableItem::RangeComplete => { collector.set_range_complete(); @@ -382,6 +396,7 @@ where } } let ret = serde_json::to_value(collector.result()?)?; + info!("Total duration: {:?}", total_duration); Ok(ret) } diff --git a/disk/src/merge.rs b/disk/src/merge.rs index 5676120..5fed55d 100644 --- a/disk/src/merge.rs +++ b/disk/src/merge.rs @@ -157,6 +157,9 @@ where Ready(Some(Ok(StreamItem::Log(item)))) } else if let Some(item) = self.stats_items.pop_front() { Ready(Some(Ok(StreamItem::Stats(item)))) + } else if self.range_complete_observed_all_emitted { + self.completed = true; + Ready(None) } else if self.data_emit_complete { if self.range_complete_observed_all { if self.range_complete_observed_all_emitted { @@ -213,10 +216,16 @@ where continue 'outer; } } else { - assert!(lowest_ts >= self.ts_last_emit); + // TODO unordered cases + if lowest_ts < self.ts_last_emit { + self.errored = true; + let msg = format!("unordered event at lowest_ts {}", lowest_ts); + return Ready(Some(Err(Error::with_msg(msg)))); + } else { + self.ts_last_emit = self.ts_last_emit.max(lowest_ts); + } { let batch = self.batch.take(); - self.ts_last_emit = lowest_ts; let rix = self.ixs[lowest_ix]; match &self.current[lowest_ix] { MergedCurVal::Val(val) => { diff --git a/httpret/src/channelarchiver.rs b/httpret/src/channelarchiver.rs index 91e8ad9..d335550 100644 --- a/httpret/src/channelarchiver.rs +++ b/httpret/src/channelarchiver.rs @@ -1,8 +1,26 @@ use crate::response; use err::Error; +use futures_core::Stream; +use futures_util::StreamExt; use http::{header, Method, Request, Response, StatusCode}; use hyper::Body; -use netpod::{log::*, NodeConfigCached, APP_JSON_LINES}; +use netpod::log::*; +use netpod::{NodeConfigCached, APP_JSON_LINES}; +use serde::Serialize; + +fn json_lines_stream(stream: S) -> impl Stream, Error>> +where + S: Stream>, + I: Serialize, +{ + stream.map(|k| { + k.map(|k| { + let mut a = serde_json::to_vec(&k).unwrap(); + a.push(0xa); + a + }) + }) +} pub struct ListIndexFilesHttpFunction {} @@ -35,7 +53,7 @@ impl ListIndexFilesHttpFunction { .ok_or(Error::with_msg_no_trace( "this node is not configured as channel archiver", ))?; - let s = archapp_wrap::archapp::archeng::list_index_files(conf); + let s = archapp_wrap::archapp::archeng::indexfiles::list_index_files(conf); let s = futures_util::stream::unfold(s, |mut st| async move { use futures_util::StreamExt; let x = st.next().await; @@ -60,6 +78,84 @@ impl ListIndexFilesHttpFunction { } } +pub struct ScanIndexFiles {} + +impl ScanIndexFiles { + pub fn prefix() -> &'static str { + "/api/4/channelarchiver/scan/indexfiles" + } + + pub fn name() -> &'static str { + "ScanIndexFiles" + } + + pub fn should_handle(path: &str) -> Option { + if path.starts_with(Self::prefix()) { + Some(Self {}) + } else { + None + } + } + + pub async fn handle(&self, req: Request, node_config: &NodeConfigCached) -> Result, Error> { + if req.method() != Method::GET { + return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?); + } + info!("{} handle uri: {:?}", Self::name(), req.uri()); + let conf = node_config + .node + .channel_archiver + .as_ref() + .ok_or(Error::with_msg_no_trace( + "this node is not configured as channel archiver", + ))?; + let s = archapp_wrap::archapp::archeng::indexfiles::scan_index_files(conf.clone()); + let s = json_lines_stream(s); + Ok(response(StatusCode::OK) + .header(header::CONTENT_TYPE, APP_JSON_LINES) + .body(Body::wrap_stream(s))?) + } +} + +pub struct ScanChannels {} + +impl ScanChannels { + pub fn prefix() -> &'static str { + "/api/4/channelarchiver/scan/channels" + } + + pub fn name() -> &'static str { + "ScanChannels" + } + + pub fn should_handle(path: &str) -> Option { + if path.starts_with(Self::prefix()) { + Some(Self {}) + } else { + None + } + } + + pub async fn handle(&self, req: Request, node_config: &NodeConfigCached) -> Result, Error> { + if req.method() != Method::GET { + return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?); + } + info!("{} handle uri: {:?}", Self::name(), req.uri()); + let conf = node_config + .node + .channel_archiver + .as_ref() + .ok_or(Error::with_msg_no_trace( + "this node is not configured as channel archiver", + ))?; + let s = archapp_wrap::archapp::archeng::indexfiles::scan_channels(conf.clone()); + let s = json_lines_stream(s); + Ok(response(StatusCode::OK) + .header(header::CONTENT_TYPE, APP_JSON_LINES) + .body(Body::wrap_stream(s))?) + } +} + pub struct ListChannelsHttpFunction {} impl ListChannelsHttpFunction { diff --git a/httpret/src/lib.rs b/httpret/src/lib.rs index 81d2a75..06c5fbe 100644 --- a/httpret/src/lib.rs +++ b/httpret/src/lib.rs @@ -286,6 +286,10 @@ async fn http_service_try(req: Request, node_config: &NodeConfigCached) -> h.handle(req, &node_config).await } else if let Some(h) = channelarchiver::ListChannelsHttpFunction::should_handle(path) { h.handle(req, &node_config).await + } else if let Some(h) = channelarchiver::ScanIndexFiles::should_handle(path) { + h.handle(req, &node_config).await + } else if let Some(h) = channelarchiver::ScanChannels::should_handle(path) { + h.handle(req, &node_config).await } else if path.starts_with("/api/1/requestStatus/") { info!("{}", path); Ok(response(StatusCode::OK).body(Body::from("{}"))?) diff --git a/items/src/lib.rs b/items/src/lib.rs index 2df1ed8..09a1e05 100644 --- a/items/src/lib.rs +++ b/items/src/lib.rs @@ -4,8 +4,8 @@ use bytes::BytesMut; use chrono::{TimeZone, Utc}; use err::Error; use netpod::timeunits::{MS, SEC}; -use netpod::RangeFilterStats; use netpod::{log::Level, AggKind, EventDataReadStats, EventQueryJsonStringFrame, NanoRange, Shape}; +use netpod::{DiskStats, RangeFilterStats}; use serde::de::{self, DeserializeOwned, Visitor}; use serde::{Deserialize, Serialize, Serializer}; use std::fmt; @@ -52,6 +52,7 @@ pub enum RangeCompletableItem { pub enum StatsItem { EventDataReadStats(EventDataReadStats), RangeFilterStats(RangeFilterStats), + DiskStats(DiskStats), } #[derive(Debug, Serialize, Deserialize)] @@ -437,3 +438,23 @@ pub trait TimeBinnableTypeAggregator: Send { fn ingest(&mut self, item: &Self::Input); fn result_reset(&mut self, range: NanoRange, expand: bool) -> Self::Output; } + +pub trait TimestampInspectable: WithTimestamps + WithLen {} + +impl TimestampInspectable for T where T: WithTimestamps + WithLen {} + +pub fn inspect_timestamps(events: &dyn TimestampInspectable, range: NanoRange) -> String { + use fmt::Write; + let rd = range.delta(); + let mut buf = String::new(); + let n = events.len(); + for i in 0..n { + if i < 3 || i > (n - 4) { + let ts = events.ts(i); + let z = ts - range.beg; + let z = z as f64 / rd as f64 * 2.0 - 1.0; + write!(&mut buf, "i {:3} tt {:6.3}\n", i, z).unwrap(); + } + } + buf +} diff --git a/netpod/src/lib.rs b/netpod/src/lib.rs index a1eda85..c4737ad 100644 --- a/netpod/src/lib.rs +++ b/netpod/src/lib.rs @@ -126,6 +126,7 @@ pub struct ArchiverAppliance { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ChannelArchiver { pub data_base_paths: Vec, + pub database: Database, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -298,6 +299,12 @@ pub struct Nanos { pub ns: u64, } +impl Nanos { + pub fn from_ns(ns: u64) -> Self { + Self { ns } + } +} + impl fmt::Debug for Nanos { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let ts = chrono::Utc.timestamp((self.ns / SEC) as i64, (self.ns % SEC) as u32); @@ -970,6 +977,58 @@ impl RangeFilterStats { } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DiskStats { + OpenStats(OpenStats), + SeekStats(SeekStats), + ReadStats(ReadStats), + ReadExactStats(ReadExactStats), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OpenStats { + pub duration: Duration, +} + +impl OpenStats { + pub fn new(duration: Duration) -> Self { + Self { duration } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SeekStats { + pub duration: Duration, +} + +impl SeekStats { + pub fn new(duration: Duration) -> Self { + Self { duration } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReadStats { + pub duration: Duration, +} + +impl ReadStats { + pub fn new(duration: Duration) -> Self { + Self { duration } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReadExactStats { + pub duration: Duration, +} + +impl ReadExactStats { + pub fn new(duration: Duration) -> Self { + Self { duration } + } +} + #[derive(Clone, Debug)] pub struct PerfOpts { pub inmem_bufcap: usize, diff --git a/streams/src/rangefilter.rs b/streams/src/rangefilter.rs index 37d317e..fb61ff4 100644 --- a/streams/src/rangefilter.rs +++ b/streams/src/rangefilter.rs @@ -1,9 +1,10 @@ +use err::Error; use futures_core::Stream; use futures_util::StreamExt; use items::StatsItem; use items::{Appendable, Clearable, PushableIndex, RangeCompletableItem, Sitemty, StreamItem, WithTimestamps}; -use netpod::NanoRange; use netpod::{log::*, RangeFilterStats}; +use netpod::{NanoRange, Nanos}; use std::pin::Pin; use std::task::{Context, Poll}; @@ -12,6 +13,7 @@ pub struct RangeFilter { range: NanoRange, range_str: String, expand: bool, + ts_max: u64, stats: RangeFilterStats, prerange: Option, have_pre: bool, @@ -34,6 +36,7 @@ where range_str: format!("{:?}", range), range, expand, + ts_max: 0, stats: RangeFilterStats::new(), prerange: None, have_pre: false, @@ -79,6 +82,19 @@ where let mut ret = item.empty_like_self(); for i1 in 0..item.len() { let ts = item.ts(i1); + if ts < self.ts_max { + self.done = true; + let msg = format!( + "unordered event i1 {} / {} ts {:?} ts_max {:?}", + i1, + item.len(), + Nanos::from_ns(ts), + Nanos::from_ns(self.ts_max) + ); + error!("{}", msg); + return Ready(Some(Err(Error::with_msg(msg)))); + } + self.ts_max = ts; if ts < self.range.beg { if self.expand { let mut prerange = if let Some(prerange) = self.prerange.take() { diff --git a/taskrun/src/append.rs b/taskrun/src/append.rs index 2a3d2ee..5425f9d 100644 --- a/taskrun/src/append.rs +++ b/taskrun/src/append.rs @@ -231,6 +231,9 @@ pub fn append_inner(dirname: &str, mut stdin: Stdin, _stderr: Stderr) -> Result< fout = next_file(&dir)?; }; } + if n1 == 0 { + break Ok(()); + } } } diff --git a/taskrun/src/lib.rs b/taskrun/src/lib.rs index 73c8ebb..09cb241 100644 --- a/taskrun/src/lib.rs +++ b/taskrun/src/lib.rs @@ -90,7 +90,14 @@ pub fn tracing_init() { .with_thread_names(true) //.with_max_level(tracing::Level::INFO) .with_env_filter(tracing_subscriber::EnvFilter::new( - ["info", "archapp::archeng=info", "daqbuffer::test=trace"].join(","), + [ + "info", + "archapp::archeng=info", + "archapp::archeng::datablockstream=info", + "archapp::storagemerge=info", + "daqbuffer::test=trace", + ] + .join(","), )) .init(); *g = 1;