daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Dominik Werder
2021-11-05 21:22:23 +01:00
parent daf3f6c14c
commit 8c7dbf9ed3
33 changed files with 527 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
archapp/src/archeng.rs
View File

@@ -9,189 +9,23 @@ pub mod diskio;
pub mod indexfiles;
pub mod indextree;
pub mod pipe;
pub mod ringbuf;
use self::indexfiles::list_index_files;
use self::indextree::channel_list;
use crate::eventsitem::EventsItem;
use crate::timed::Timed;
use crate::wrap_task;
use async_channel::{Receiver, Sender};
use commonio::StatsChannel;
use err::Error;
use futures_util::StreamExt;
use items::{Sitemty, StatsItem, StreamItem, WithLen};
use items::{StreamItem, WithLen};
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{
ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse, DiskStats, OpenStats, ReadExactStats, ReadStats,
SeekStats,
};
use netpod::{ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse};
use serde::Serialize;
use std::convert::TryInto;
use std::fmt;
use std::io::{self, SeekFrom};
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Instant;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncReadExt, AsyncSeekExt};
/*
struct ReadExactWrap<'a> {
fut: &'a mut dyn Future<Output = io::Result<usize>>,
}
trait TimedIo {
fn read_exact<'a, F>(&'a mut self, buf: &'a mut [u8]) -> ReadExactWrap
where
Self: Unpin;
}
impl TimedIo for File {
fn read_exact<'a, F>(&'a mut self, buf: &'a mut [u8]) -> ReadExactWrap
where
Self: Unpin,
{
let fut = tokio::io::AsyncReadExt::read_exact(self, buf);
ReadExactWrap { fut: Box::pin(fut) }
}
}
*/
const EPICS_EPOCH_OFFSET: u64 = 631152000 * SEC;
const LOG_IO: bool = true;
const STATS_IO: bool = true;
static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0);
fn channel_send_error() {
let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel);
if c < 10 {
error!("CHANNEL_SEND_ERROR {}", c);
}
}
pub struct StatsChannel {
chn: Sender<Sitemty<EventsItem>>,
}
impl fmt::Debug for StatsChannel {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("StatsChannel").finish()
}
}
impl StatsChannel {
pub fn new(chn: Sender<Sitemty<EventsItem>>) -> Self {
Self { chn }
}
pub fn dummy() -> Self {
let (tx, rx) = async_channel::bounded(2);
taskrun::spawn(async move {
let mut rx = rx;
while let Some(_) = rx.next().await {}
});
Self::new(tx)
}
pub async fn send(&self, item: StatsItem) -> Result<(), Error> {
Ok(self.chn.send(Ok(StreamItem::Stats(item))).await?)
}
}
impl Clone for StatsChannel {
fn clone(&self) -> Self {
Self { chn: self.chn.clone() }
}
}
pub async fn open_read(path: PathBuf, stats: &StatsChannel) -> io::Result<File> {
let ts1 = Instant::now();
let res = OpenOptions::new().read(true).open(path).await;
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
if LOG_IO {
let dt = dt.as_secs_f64() * 1e3;
debug!("timed open_read dt: {:.3} ms", dt);
}
if STATS_IO {
if let Err(_) = stats
.send(StatsItem::DiskStats(DiskStats::OpenStats(OpenStats::new(
ts2.duration_since(ts1),
))))
.await
{
channel_send_error();
}
}
res
}
async fn seek(file: &mut File, pos: SeekFrom, stats: &StatsChannel) -> io::Result<u64> {
let ts1 = Instant::now();
let res = file.seek(pos).await;
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
if LOG_IO {
let dt = dt.as_secs_f64() * 1e3;
debug!("timed seek dt: {:.3} ms", dt);
}
if STATS_IO {
if let Err(_) = stats
.send(StatsItem::DiskStats(DiskStats::SeekStats(SeekStats::new(
ts2.duration_since(ts1),
))))
.await
{
channel_send_error();
}
}
res
}
async fn read(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result<usize> {
let ts1 = Instant::now();
let res = file.read(buf).await;
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
if LOG_IO {
let dt = dt.as_secs_f64() * 1e3;
debug!("timed read dt: {:.3} ms res: {:?}", dt, res);
}
if STATS_IO {
if let Err(_) = stats
.send(StatsItem::DiskStats(DiskStats::ReadStats(ReadStats::new(
ts2.duration_since(ts1),
))))
.await
{
channel_send_error();
}
}
res
}
async fn read_exact(file: &mut File, buf: &mut [u8], stats: &StatsChannel) -> io::Result<usize> {
let ts1 = Instant::now();
let res = file.read_exact(buf).await;
let ts2 = Instant::now();
let dt = ts2.duration_since(ts1);
if LOG_IO {
let dt = dt.as_secs_f64() * 1e3;
debug!("timed read_exact dt: {:.3} ms res: {:?}", dt, res);
}
if STATS_IO {
if let Err(_) = stats
.send(StatsItem::DiskStats(DiskStats::ReadExactStats(ReadExactStats::new(
ts2.duration_since(ts1),
))))
.await
{
channel_send_error();
};
}
res
}
pub fn name_hash(s: &str, ht_len: u32) -> u32 {
let mut h = 0;
@@ -334,7 +168,6 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R
let stream = Box::pin(stream);
let stream = blockstream::BlockStream::new(stream, q.range.clone(), 1);
let mut stream = stream;
let timed_expand = Timed::new("channel_config EXPAND");
while let Some(item) = stream.next().await {
use blockstream::BlockItem::*;
match item {
@@ -346,7 +179,7 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R
}
}
JsVal(jsval) => {
if false {
if true {
info!("jsval: {}", serde_json::to_string(&jsval)?);
}
}
@@ -357,7 +190,6 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R
}
}
}
drop(timed_expand);
if type_info.is_none() {
let timed_normal = Timed::new("channel_config NORMAL");
warn!("channel_config expand mode returned none");
@@ -376,7 +208,7 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R
}
}
JsVal(jsval) => {
if false {
if true {
info!("jsval: {}", serde_json::to_string(&jsval)?);
}
}
@@ -406,7 +238,8 @@ pub async fn channel_config(q: &ChannelConfigQuery, conf: &ChannelArchiver) -> R
mod test {
use crate::archeng::datablock::{read_data_1, read_datafile_header};
use crate::archeng::indextree::{read_channel, read_datablockref, search_record};
use crate::archeng::{open_read, StatsChannel, EPICS_EPOCH_OFFSET};
use crate::archeng::{StatsChannel, EPICS_EPOCH_OFFSET};
use commonio::open_read;
use err::Error;
use netpod::log::*;
use netpod::timeunits::*;

2
archapp/src/archeng/backreadbuf.rs
View File

@@ -1,4 +1,4 @@
use crate::archeng::{read, seek, StatsChannel};
use commonio::{read, seek, StatsChannel};
use err::Error;
use netpod::log::*;
use std::borrow::BorrowMut;

9
archapp/src/archeng/blockrefstream.rs
View File

@@ -4,8 +4,8 @@ use crate::archeng::indexfiles::{database_connect, unfold_stream, UnfoldExec};
use crate::archeng::indextree::{
read_datablockref2, DataheaderPos, Dataref, HeaderVersion, IndexFileBasics, RecordIter, RecordTarget,
};
use crate::archeng::ringbuf::RingBuf;
use crate::archeng::{open_read, StatsChannel};
use commonio::ringbuf::RingBuf;
use commonio::{open_read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
use items::WithLen;
@@ -82,7 +82,10 @@ impl BlockrefStream {
match self.steps {
Start => {
self.steps = SelectIndexFile;
Ok(Some((BlockrefItem::JsVal(JsVal::String(format!("START"))), self)))
Ok(Some((
BlockrefItem::JsVal(JsVal::String(format!("{} START", module_path!()))),
self,
)))
}
SelectIndexFile => {
let dbc = database_connect(&self.conf.database).await?;

8
archapp/src/archeng/blockstream.rs
View File

@@ -1,13 +1,13 @@
use super::indextree::DataheaderPos;
use super::ringbuf::RingBuf;
use super::{open_read, StatsChannel};
use crate::archeng::blockrefstream::BlockrefItem;
use crate::archeng::datablock::{read_data2, read_datafile_header2};
use crate::eventsitem::EventsItem;
use crate::archeng::indextree::DataheaderPos;
use commonio::ringbuf::RingBuf;
use commonio::{open_read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
use futures_util::stream::FuturesOrdered;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::{WithLen, WithTimestamps};
use netpod::{log::*, NanoRange};
use serde::Serialize;

8
archapp/src/archeng/configs.rs
View File

@@ -76,10 +76,11 @@ impl Stream for ChannelNameStream {
Ready(Ok(dbc)) => {
self.connect_fut = None;
let off = self.off as i64;
info!("select channels off {}", off);
let fut = async move {
let rows = dbc
.query(
"select rowid, name from channels where config = '{}'::jsonb order by name offset $1 limit 1000",
"select rowid, name from channels where config = '{}'::jsonb order by name offset $1 limit 64",
&[&off],
)
.await?;
@@ -235,7 +236,10 @@ impl Stream for ConfigStream {
match fut.await {
Ok(Ok(k)) => Ok(Res::Response(k)),
Ok(Err(e)) => Err(e),
Err(_) => Ok(Res::TimedOut(q.channel.name)),
Err(_) => {
warn!("timeout");
Ok(Res::TimedOut(q.channel.name))
}
}
};
self.get_fut = Some(Box::pin(fut));

12
archapp/src/archeng/datablock.rs
View File

@@ -1,11 +1,11 @@
use super::format_hex_block;
use super::indextree::DataheaderPos;
use crate::archeng::ringbuf::RingBuf;
use crate::archeng::{read_exact, read_string, readf64, readu16, readu32, seek, StatsChannel, EPICS_EPOCH_OFFSET};
use crate::eventsitem::EventsItem;
use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use crate::archeng::indextree::DataheaderPos;
use crate::archeng::{format_hex_block, read_string, readf64, readu16, readu32, StatsChannel, EPICS_EPOCH_OFFSET};
use commonio::ringbuf::RingBuf;
use commonio::{read_exact, seek};
use err::Error;
use items::eventsitem::EventsItem;
use items::eventvalues::EventValues;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::waveevents::WaveEvents;
use netpod::log::*;
use netpod::timeunits::SEC;

6
archapp/src/archeng/datablockstream.rs
View File

@@ -1,14 +1,14 @@
use crate::archeng::datablock::{read_data_1, read_datafile_header};
use crate::archeng::indexfiles::index_file_path_list;
use crate::archeng::indextree::{read_channel, read_datablockref, search_record, search_record_expand, DataheaderPos};
use crate::archeng::{open_read, StatsChannel};
use crate::eventsitem::EventsItem;
use crate::storagemerge::StorageMerge;
use crate::timed::Timed;
use async_channel::{Receiver, Sender};
use commonio::{open_read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
use futures_util::{FutureExt, StreamExt};
use items::eventsitem::EventsItem;
use items::{inspect_timestamps, RangeCompletableItem, Sitemty, StreamItem, WithLen};
use netpod::log::*;
use netpod::{Channel, NanoRange};
@@ -317,10 +317,10 @@ impl Stream for DatablockStream {
#[cfg(test)]
mod test {
use super::DatablockStream;
use crate::eventsitem::EventsItem;
use chrono::{DateTime, Utc};
use err::Error;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::{LogItem, Sitemty, StatsItem, StreamItem};
use netpod::timeunits::SEC;
use netpod::{log::*, RangeFilterStats};

2
archapp/src/archeng/indexfiles.rs
View File

@@ -1,7 +1,7 @@
use crate::archeng::{open_read, read, StatsChannel};
use crate::timed::Timed;
use crate::wrap_task;
use async_channel::Receiver;
use commonio::{open_read, read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
use futures_util::stream::unfold;

10
archapp/src/archeng/indextree.rs
View File

@@ -1,7 +1,6 @@
use crate::archeng::ringbuf::RingBuf;
use crate::archeng::{
format_hex_block, name_hash, open_read, readu16, readu32, readu64, StatsChannel, EPICS_EPOCH_OFFSET,
};
use crate::archeng::{format_hex_block, name_hash, readu16, readu32, readu64, StatsChannel, EPICS_EPOCH_OFFSET};
use commonio::open_read;
use commonio::ringbuf::RingBuf;
use err::Error;
use netpod::{log::*, NanoRange};
use netpod::{timeunits::SEC, FilePos, Nanos};
@@ -1048,7 +1047,8 @@ mod test {
use crate::archeng::indextree::{
read_channel, read_datablockref, read_file_basics, search_record, IndexFileBasics,
};
use crate::archeng::{open_read, StatsChannel, EPICS_EPOCH_OFFSET};
use crate::archeng::EPICS_EPOCH_OFFSET;
use commonio::{open_read, StatsChannel};
use err::Error;
#[allow(unused)]
use netpod::log::*;

154
archapp/src/archeng/ringbuf.rs
View File

@@ -1,154 +0,0 @@
use crate::archeng::{read, seek, StatsChannel};
use err::Error;
use netpod::log::*;
use std::fmt;
use std::{borrow::BorrowMut, io::SeekFrom};
use tokio::fs::File;
pub struct RingBuf<F> {
file: Option<F>,
buf: Vec<u8>,
abs: usize,
wp: usize,
rp: usize,
stats: StatsChannel,
seek_request: u64,
seek_done: u64,
read_done: u64,
small_pos: u64,
small_neg: u64,
bytes_read: u64,
}
impl<F> RingBuf<F>
where
F: BorrowMut<File>,
{
pub async fn new(file: F, pos: u64, stats: StatsChannel) -> Result<Self, Error> {
let mut ret = Self {
file: Some(file),
buf: vec![0; 1024 * 1024],
abs: usize::MAX,
wp: 0,
rp: 0,
stats,
seek_request: 0,
seek_done: 0,
read_done: 0,
small_pos: 0,
small_neg: 0,
bytes_read: 0,
};
ret.seek(pos).await?;
Ok(ret)
}
pub fn into_file(mut self) -> F {
self.file.take().unwrap()
}
pub fn len(&self) -> usize {
self.wp - self.rp
}
pub fn adv(&mut self, n: usize) {
self.rp += n;
}
pub fn data(&self) -> &[u8] {
&self.buf[self.rp..self.wp]
}
async fn fill(&mut self) -> Result<usize, Error> {
if self.rp == self.wp {
if self.rp != 0 {
self.wp = 0;
self.rp = 0;
}
} else {
unsafe {
std::ptr::copy::<u8>(&self.buf[self.rp], &mut self.buf[0], self.len());
self.wp -= self.rp;
self.rp = 0;
}
}
let max = (self.buf.len() - self.wp).min(1024 * 8) + self.wp;
let n = read(
self.file.as_mut().unwrap().borrow_mut(),
&mut self.buf[self.wp..max],
&self.stats,
)
.await?;
self.wp += n;
self.read_done += 1;
self.bytes_read += n as u64;
Ok(n)
}
pub async fn fill_min(&mut self, min: usize) -> Result<usize, Error> {
let len = self.len();
while self.len() < min {
let n = self.fill().await?;
if n == 0 {
return Err(Error::with_msg_no_trace(format!("fill_min can not read min {}", min)));
}
}
Ok(self.len() - len)
}
pub async fn seek(&mut self, pos: u64) -> Result<u64, Error> {
let dp = pos as i64 - self.rp_abs() as i64;
if dp < 0 && dp > -2048 {
debug!("small NEG seek {}", dp);
} else if dp == 0 {
// TODO check callsites, some cases could be eliminated.
//debug!("zero seek");
return Ok(pos);
} else if dp > 0 && dp < 2048 {
debug!("small POS seek {}", dp);
}
self.abs = pos as usize;
self.rp = 0;
self.wp = 0;
let ret = seek(
self.file.as_mut().unwrap().borrow_mut(),
SeekFrom::Start(pos),
&self.stats,
)
.await
.map_err(|e| Error::from(e))?;
self.seek_request += 1;
self.seek_done += 1;
Ok(ret)
}
pub fn rp_abs(&self) -> u64 {
self.abs as u64 + self.rp as u64
}
pub fn bytes_read(&self) -> u64 {
self.bytes_read
}
}
impl<F> fmt::Debug for RingBuf<F> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("RingBuf")
.field("abs", &self.abs)
.field("wp", &self.wp)
.field("rp", &self.rp)
.field("seek_request", &self.seek_request)
.field("seek_done", &self.seek_done)
.field("read_done", &self.read_done)
.field("small_pos", &self.small_pos)
.field("small_neg", &self.small_neg)
.field("bytes_read", &self.bytes_read)
.finish()
}
}
impl<F> Drop for RingBuf<F> {
fn drop(&mut self) {
trace!("Drop {:?}", self);
}
}

465
archapp/src/binnedevents.rs
View File

@@ -1,465 +0,0 @@
use items::{
xbinnedscalarevents::XBinnedScalarEvents, xbinnedwaveevents::XBinnedWaveEvents, Appendable, Clearable,
PushableIndex, WithLen, WithTimestamps,
};
use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape};
use crate::{
eventsitem::EventsItem,
plainevents::{PlainEvents, ScalarPlainEvents},
};
#[derive(Debug)]
pub enum SingleBinWaveEvents {
Byte(XBinnedScalarEvents<i8>),
Short(XBinnedScalarEvents<i16>),
Int(XBinnedScalarEvents<i32>),
Float(XBinnedScalarEvents<f32>),
Double(XBinnedScalarEvents<f64>),
}
impl SingleBinWaveEvents {
pub fn variant_name(&self) -> String {
use SingleBinWaveEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use SingleBinWaveEvents::*;
match self {
Byte(k) => match ak {
AggKind::EventBlobs => panic!(),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::Byte(k))),
AggKind::TimeWeightedScalar => err::todoval(),
AggKind::DimXBins1 => err::todoval(),
AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())),
},
_ => err::todoval(),
}
}
}
impl Clearable for SingleBinWaveEvents {
fn clear(&mut self) {
match self {
SingleBinWaveEvents::Byte(k) => k.clear(),
SingleBinWaveEvents::Short(k) => k.clear(),
SingleBinWaveEvents::Int(k) => k.clear(),
SingleBinWaveEvents::Float(k) => k.clear(),
SingleBinWaveEvents::Double(k) => k.clear(),
}
}
}
impl Appendable for SingleBinWaveEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for SingleBinWaveEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for SingleBinWaveEvents {
fn len(&self) -> usize {
use SingleBinWaveEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
}
}
}
impl WithTimestamps for SingleBinWaveEvents {
fn ts(&self, ix: usize) -> u64 {
use SingleBinWaveEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
}
}
}
impl HasShape for SingleBinWaveEvents {
fn shape(&self) -> Shape {
use SingleBinWaveEvents::*;
match self {
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
}
}
}
impl HasScalarType for SingleBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use SingleBinWaveEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
#[derive(Debug)]
pub enum MultiBinWaveEvents {
Byte(XBinnedWaveEvents<i8>),
Short(XBinnedWaveEvents<i16>),
Int(XBinnedWaveEvents<i32>),
Float(XBinnedWaveEvents<f32>),
Double(XBinnedWaveEvents<f64>),
}
impl MultiBinWaveEvents {
pub fn variant_name(&self) -> String {
use MultiBinWaveEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use MultiBinWaveEvents::*;
match self {
Byte(k) => match ak {
AggKind::EventBlobs => panic!(),
AggKind::Plain => EventsItem::XBinnedEvents(XBinnedEvents::MultiBinWave(MultiBinWaveEvents::Byte(k))),
AggKind::TimeWeightedScalar => err::todoval(),
AggKind::DimXBins1 => err::todoval(),
AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())),
},
_ => err::todoval(),
}
}
}
impl Clearable for MultiBinWaveEvents {
fn clear(&mut self) {
match self {
MultiBinWaveEvents::Byte(k) => k.clear(),
MultiBinWaveEvents::Short(k) => k.clear(),
MultiBinWaveEvents::Int(k) => k.clear(),
MultiBinWaveEvents::Float(k) => k.clear(),
MultiBinWaveEvents::Double(k) => k.clear(),
}
}
}
impl Appendable for MultiBinWaveEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for MultiBinWaveEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for MultiBinWaveEvents {
fn len(&self) -> usize {
use MultiBinWaveEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
}
}
}
impl WithTimestamps for MultiBinWaveEvents {
fn ts(&self, ix: usize) -> u64 {
use MultiBinWaveEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
}
}
}
impl HasShape for MultiBinWaveEvents {
fn shape(&self) -> Shape {
use MultiBinWaveEvents::*;
match self {
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
}
}
}
impl HasScalarType for MultiBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use MultiBinWaveEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
#[derive(Debug)]
pub enum XBinnedEvents {
Scalar(ScalarPlainEvents),
SingleBinWave(SingleBinWaveEvents),
MultiBinWave(MultiBinWaveEvents),
}
impl XBinnedEvents {
pub fn variant_name(&self) -> String {
use XBinnedEvents::*;
match self {
Scalar(h) => format!("Scalar({})", h.variant_name()),
SingleBinWave(h) => format!("SingleBinWave({})", h.variant_name()),
MultiBinWave(h) => format!("MultiBinWave({})", h.variant_name()),
}
}
pub fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use XBinnedEvents::*;
match self {
Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)),
SingleBinWave(k) => k.x_aggregate(ak),
MultiBinWave(k) => k.x_aggregate(ak),
}
}
}
impl Clearable for XBinnedEvents {
fn clear(&mut self) {
match self {
XBinnedEvents::Scalar(k) => k.clear(),
XBinnedEvents::SingleBinWave(k) => k.clear(),
XBinnedEvents::MultiBinWave(k) => k.clear(),
}
}
}
impl Appendable for XBinnedEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Scalar(k) => Self::Scalar(k.empty_like_self()),
Self::SingleBinWave(k) => Self::SingleBinWave(k.empty_like_self()),
Self::MultiBinWave(k) => Self::MultiBinWave(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Scalar(k) => match src {
Self::Scalar(j) => k.append(j),
_ => panic!(),
},
Self::SingleBinWave(k) => match src {
Self::SingleBinWave(j) => k.append(j),
_ => panic!(),
},
Self::MultiBinWave(k) => match src {
Self::MultiBinWave(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for XBinnedEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Scalar(k) => match src {
Self::Scalar(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::SingleBinWave(k) => match src {
Self::SingleBinWave(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::MultiBinWave(k) => match src {
Self::MultiBinWave(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for XBinnedEvents {
fn len(&self) -> usize {
use XBinnedEvents::*;
match self {
Scalar(j) => j.len(),
SingleBinWave(j) => j.len(),
MultiBinWave(j) => j.len(),
}
}
}
impl WithTimestamps for XBinnedEvents {
fn ts(&self, ix: usize) -> u64 {
use XBinnedEvents::*;
match self {
Scalar(j) => j.ts(ix),
SingleBinWave(j) => j.ts(ix),
MultiBinWave(j) => j.ts(ix),
}
}
}
impl HasShape for XBinnedEvents {
fn shape(&self) -> Shape {
use XBinnedEvents::*;
match self {
Scalar(h) => h.shape(),
SingleBinWave(h) => h.shape(),
MultiBinWave(h) => h.shape(),
}
}
}
impl HasScalarType for XBinnedEvents {
fn scalar_type(&self) -> ScalarType {
use XBinnedEvents::*;
match self {
Scalar(h) => h.scalar_type(),
SingleBinWave(h) => h.scalar_type(),
MultiBinWave(h) => h.scalar_type(),
}
}
}

6
archapp/src/events.rs
View File

@@ -1,15 +1,15 @@
use crate::binnedevents::{MultiBinWaveEvents, SingleBinWaveEvents, XBinnedEvents};
use crate::eventsitem::EventsItem;
use crate::generated::EPICSEvent::PayloadType;
use crate::parse::multi::parse_all_ts;
use crate::parse::PbFileReader;
use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use crate::storagemerge::StorageMerge;
use chrono::{TimeZone, Utc};
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::binnedevents::{MultiBinWaveEvents, SingleBinWaveEvents, XBinnedEvents};
use items::eventsitem::EventsItem;
use items::eventvalues::EventValues;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::waveevents::WaveEvents;
use items::xbinnedscalarevents::XBinnedScalarEvents;
use items::xbinnedwaveevents::XBinnedWaveEvents;

156
archapp/src/eventsitem.rs
View File

@@ -1,156 +0,0 @@
use items::{Appendable, Clearable, PushableIndex, WithLen, WithTimestamps};
use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape};
use crate::{
binnedevents::XBinnedEvents,
plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents},
};
#[derive(Debug)]
pub enum EventsItem {
Plain(PlainEvents),
XBinnedEvents(XBinnedEvents),
}
impl EventsItem {
pub fn is_wave(&self) -> bool {
use EventsItem::*;
match self {
Plain(h) => h.is_wave(),
XBinnedEvents(h) => {
if let Shape::Wave(_) = h.shape() {
true
} else {
false
}
}
}
}
pub fn variant_name(&self) -> String {
use EventsItem::*;
match self {
Plain(h) => format!("Plain({})", h.variant_name()),
XBinnedEvents(h) => format!("Plain({})", h.variant_name()),
}
}
pub fn x_aggregate(self, ak: &AggKind) -> Self {
use EventsItem::*;
match self {
Plain(k) => k.x_aggregate(ak),
XBinnedEvents(k) => k.x_aggregate(ak),
}
}
pub fn type_info(&self) -> (ScalarType, Shape) {
match self {
EventsItem::Plain(k) => match k {
PlainEvents::Scalar(k) => match k {
ScalarPlainEvents::Byte(_) => (ScalarType::I8, Shape::Scalar),
ScalarPlainEvents::Short(_) => (ScalarType::I16, Shape::Scalar),
ScalarPlainEvents::Int(_) => (ScalarType::I32, Shape::Scalar),
ScalarPlainEvents::Float(_) => (ScalarType::F32, Shape::Scalar),
ScalarPlainEvents::Double(_) => (ScalarType::F64, Shape::Scalar),
},
PlainEvents::Wave(k) => match k {
// TODO
// Inherent issue for the non-static-type backends:
// there is a chance that we can't determine the shape here.
WavePlainEvents::Byte(k) => (ScalarType::I8, k.shape().unwrap()),
WavePlainEvents::Short(k) => (ScalarType::I16, k.shape().unwrap()),
WavePlainEvents::Int(k) => (ScalarType::I32, k.shape().unwrap()),
WavePlainEvents::Float(k) => (ScalarType::F32, k.shape().unwrap()),
WavePlainEvents::Double(k) => (ScalarType::F64, k.shape().unwrap()),
},
},
EventsItem::XBinnedEvents(_k) => panic!(),
}
}
}
impl WithLen for EventsItem {
fn len(&self) -> usize {
use EventsItem::*;
match self {
Plain(j) => j.len(),
XBinnedEvents(j) => j.len(),
}
}
}
impl WithTimestamps for EventsItem {
fn ts(&self, ix: usize) -> u64 {
use EventsItem::*;
match self {
Plain(j) => j.ts(ix),
XBinnedEvents(j) => j.ts(ix),
}
}
}
impl Appendable for EventsItem {
fn empty_like_self(&self) -> Self {
match self {
EventsItem::Plain(k) => EventsItem::Plain(k.empty_like_self()),
EventsItem::XBinnedEvents(k) => EventsItem::XBinnedEvents(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Plain(k) => match src {
Self::Plain(j) => k.append(j),
_ => panic!(),
},
Self::XBinnedEvents(k) => match src {
Self::XBinnedEvents(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for EventsItem {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Plain(k) => match src {
Self::Plain(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::XBinnedEvents(k) => match src {
Self::XBinnedEvents(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl Clearable for EventsItem {
fn clear(&mut self) {
match self {
EventsItem::Plain(k) => k.clear(),
EventsItem::XBinnedEvents(k) => k.clear(),
}
}
}
impl HasShape for EventsItem {
fn shape(&self) -> Shape {
use EventsItem::*;
match self {
Plain(h) => h.shape(),
XBinnedEvents(h) => h.shape(),
}
}
}
impl HasScalarType for EventsItem {
fn scalar_type(&self) -> ScalarType {
use EventsItem::*;
match self {
Plain(h) => h.scalar_type(),
XBinnedEvents(h) => h.scalar_type(),
}
}
}

3
archapp/src/lib.rs
View File

@@ -3,14 +3,11 @@ pub mod generated;
#[cfg(not(feature = "devread"))]
pub mod generated {}
pub mod archeng;
pub mod binnedevents;
pub mod events;
pub mod eventsitem;
#[cfg(feature = "devread")]
pub mod parse;
#[cfg(not(feature = "devread"))]
pub mod parsestub;
pub mod plainevents;
pub mod storagemerge;
#[cfg(feature = "devread")]
#[cfg(test)]

4
archapp/src/parse.rs
View File

@@ -1,15 +1,15 @@
pub mod multi;
use crate::events::parse_data_filename;
use crate::eventsitem::EventsItem;
use crate::generated::EPICSEvent::PayloadType;
use crate::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use crate::unescape_archapp_msg;
use archapp_xc::*;
use async_channel::{bounded, Receiver};
use chrono::{TimeZone, Utc};
use err::Error;
use items::eventsitem::EventsItem;
use items::eventvalues::EventValues;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::waveevents::WaveEvents;
use netpod::log::*;
use netpod::{ArchiverAppliance, ChannelConfigQuery, ChannelConfigResponse, NodeConfigCached};

465
archapp/src/plainevents.rs
View File

@@ -1,465 +0,0 @@
use crate::binnedevents::{SingleBinWaveEvents, XBinnedEvents};
use crate::eventsitem::EventsItem;
use err::Error;
use items::eventvalues::EventValues;
use items::waveevents::{WaveEvents, WaveXBinner};
use items::{Appendable, Clearable, EventsNodeProcessor, PushableIndex, WithLen, WithTimestamps};
use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape};
#[derive(Debug)]
pub enum ScalarPlainEvents {
Byte(EventValues<i8>),
Short(EventValues<i16>),
Int(EventValues<i32>),
Float(EventValues<f32>),
Double(EventValues<f64>),
}
impl ScalarPlainEvents {
pub fn variant_name(&self) -> String {
use ScalarPlainEvents::*;
match self {
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
}
impl Clearable for ScalarPlainEvents {
fn clear(&mut self) {
match self {
ScalarPlainEvents::Byte(k) => k.clear(),
ScalarPlainEvents::Short(k) => k.clear(),
ScalarPlainEvents::Int(k) => k.clear(),
ScalarPlainEvents::Float(k) => k.clear(),
ScalarPlainEvents::Double(k) => k.clear(),
}
}
}
impl Appendable for ScalarPlainEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for ScalarPlainEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for ScalarPlainEvents {
fn len(&self) -> usize {
use ScalarPlainEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
}
}
}
impl WithTimestamps for ScalarPlainEvents {
fn ts(&self, ix: usize) -> u64 {
use ScalarPlainEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
}
}
}
impl HasShape for ScalarPlainEvents {
fn shape(&self) -> Shape {
match self {
_ => Shape::Scalar,
}
}
}
impl HasScalarType for ScalarPlainEvents {
fn scalar_type(&self) -> ScalarType {
use ScalarPlainEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
#[derive(Debug)]
pub enum WavePlainEvents {
Byte(WaveEvents<i8>),
Short(WaveEvents<i16>),
Int(WaveEvents<i32>),
Float(WaveEvents<f32>),
Double(WaveEvents<f64>),
}
impl WavePlainEvents {
pub fn shape(&self) -> Result<Shape, Error> {
match self {
WavePlainEvents::Byte(k) => k.shape(),
WavePlainEvents::Short(k) => k.shape(),
WavePlainEvents::Int(k) => k.shape(),
WavePlainEvents::Float(k) => k.shape(),
WavePlainEvents::Double(k) => k.shape(),
}
}
}
macro_rules! wagg1 {
($k:expr, $ak:expr, $shape:expr, $sty:ident) => {
match $ak {
AggKind::EventBlobs => panic!(),
AggKind::Plain => EventsItem::Plain(PlainEvents::Wave(WavePlainEvents::$sty($k))),
AggKind::TimeWeightedScalar => {
let p = WaveXBinner::create($shape, $ak.clone());
let j = p.process($k);
EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j)))
}
AggKind::DimXBins1 => {
let p = WaveXBinner::create($shape, $ak.clone());
let j = p.process($k);
EventsItem::XBinnedEvents(XBinnedEvents::SingleBinWave(SingleBinWaveEvents::$sty(j)))
}
AggKind::DimXBinsN(_) => EventsItem::Plain(PlainEvents::Wave(err::todoval())),
}
};
}
impl WavePlainEvents {
pub fn variant_name(&self) -> String {
use WavePlainEvents::*;
match self {
Byte(h) => format!("Byte({})", h.vals.first().map_or(0, |j| j.len())),
Short(h) => format!("Short({})", h.vals.first().map_or(0, |j| j.len())),
Int(h) => format!("Int({})", h.vals.first().map_or(0, |j| j.len())),
Float(h) => format!("Float({})", h.vals.first().map_or(0, |j| j.len())),
Double(h) => format!("Double({})", h.vals.first().map_or(0, |j| j.len())),
}
}
fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use WavePlainEvents::*;
let shape = self.shape().unwrap();
match self {
Byte(k) => wagg1!(k, ak, shape, Byte),
Short(k) => wagg1!(k, ak, shape, Short),
Int(k) => wagg1!(k, ak, shape, Int),
Float(k) => wagg1!(k, ak, shape, Float),
Double(k) => wagg1!(k, ak, shape, Double),
}
}
}
impl Clearable for WavePlainEvents {
fn clear(&mut self) {
match self {
WavePlainEvents::Byte(k) => k.clear(),
WavePlainEvents::Short(k) => k.clear(),
WavePlainEvents::Int(k) => k.clear(),
WavePlainEvents::Float(k) => k.clear(),
WavePlainEvents::Double(k) => k.clear(),
}
}
}
impl Appendable for WavePlainEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Byte(k) => Self::Byte(k.empty_like_self()),
Self::Short(k) => Self::Short(k.empty_like_self()),
Self::Int(k) => Self::Int(k.empty_like_self()),
Self::Float(k) => Self::Float(k.empty_like_self()),
Self::Double(k) => Self::Double(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.append(j),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.append(j),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.append(j),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.append(j),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for WavePlainEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Byte(k) => match src {
Self::Byte(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Short(k) => match src {
Self::Short(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Int(k) => match src {
Self::Int(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Float(k) => match src {
Self::Float(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Double(k) => match src {
Self::Double(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for WavePlainEvents {
fn len(&self) -> usize {
use WavePlainEvents::*;
match self {
Byte(j) => j.len(),
Short(j) => j.len(),
Int(j) => j.len(),
Float(j) => j.len(),
Double(j) => j.len(),
}
}
}
impl WithTimestamps for WavePlainEvents {
fn ts(&self, ix: usize) -> u64 {
use WavePlainEvents::*;
match self {
Byte(j) => j.ts(ix),
Short(j) => j.ts(ix),
Int(j) => j.ts(ix),
Float(j) => j.ts(ix),
Double(j) => j.ts(ix),
}
}
}
impl HasShape for WavePlainEvents {
fn shape(&self) -> Shape {
/*use WavePlainEvents::*;
match self {
Byte(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Short(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Int(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Float(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
Double(h) => Shape::Wave(h.vals.first().map_or(0, |x| x.len() as u32)),
}*/
self.shape().unwrap()
}
}
impl HasScalarType for WavePlainEvents {
fn scalar_type(&self) -> ScalarType {
use WavePlainEvents::*;
match self {
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
#[derive(Debug)]
pub enum PlainEvents {
Scalar(ScalarPlainEvents),
Wave(WavePlainEvents),
}
impl PlainEvents {
pub fn is_wave(&self) -> bool {
use PlainEvents::*;
match self {
Scalar(_) => false,
Wave(_) => true,
}
}
pub fn variant_name(&self) -> String {
use PlainEvents::*;
match self {
Scalar(h) => format!("Scalar({})", h.variant_name()),
Wave(h) => format!("Scalar({})", h.variant_name()),
}
}
pub fn x_aggregate(self, ak: &AggKind) -> EventsItem {
use PlainEvents::*;
match self {
Scalar(k) => EventsItem::Plain(PlainEvents::Scalar(k)),
Wave(k) => k.x_aggregate(ak),
}
}
}
impl Clearable for PlainEvents {
fn clear(&mut self) {
match self {
PlainEvents::Scalar(k) => k.clear(),
PlainEvents::Wave(k) => k.clear(),
}
}
}
impl Appendable for PlainEvents {
fn empty_like_self(&self) -> Self {
match self {
Self::Scalar(k) => Self::Scalar(k.empty_like_self()),
Self::Wave(k) => Self::Wave(k.empty_like_self()),
}
}
fn append(&mut self, src: &Self) {
match self {
PlainEvents::Scalar(k) => match src {
Self::Scalar(j) => k.append(j),
_ => panic!(),
},
PlainEvents::Wave(k) => match src {
Self::Wave(j) => k.append(j),
_ => panic!(),
},
}
}
}
impl PushableIndex for PlainEvents {
fn push_index(&mut self, src: &Self, ix: usize) {
match self {
Self::Scalar(k) => match src {
Self::Scalar(j) => k.push_index(j, ix),
_ => panic!(),
},
Self::Wave(k) => match src {
Self::Wave(j) => k.push_index(j, ix),
_ => panic!(),
},
}
}
}
impl WithLen for PlainEvents {
fn len(&self) -> usize {
use PlainEvents::*;
match self {
Scalar(j) => j.len(),
Wave(j) => j.len(),
}
}
}
impl WithTimestamps for PlainEvents {
fn ts(&self, ix: usize) -> u64 {
use PlainEvents::*;
match self {
Scalar(j) => j.ts(ix),
Wave(j) => j.ts(ix),
}
}
}
impl HasShape for PlainEvents {
fn shape(&self) -> Shape {
use PlainEvents::*;
match self {
Scalar(h) => HasShape::shape(h),
Wave(h) => HasShape::shape(h),
}
}
}
impl HasScalarType for PlainEvents {
fn scalar_type(&self) -> ScalarType {
use PlainEvents::*;
match self {
Scalar(h) => h.scalar_type(),
Wave(h) => h.scalar_type(),
}
}
}

3
archapp/src/storagemerge.rs
View File

@@ -1,6 +1,7 @@
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::{
inspect_timestamps, Appendable, LogItem, PushableIndex, RangeCompletableItem, Sitemty, StatsItem, StreamItem,
};
@@ -11,8 +12,6 @@ use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::eventsitem::EventsItem;
/**
Priority-Merge events from different candidate sources.