Adapt empty generator

2021-10-18 21:41:35 +02:00
parent 7b79070e8a
commit dafe0a6e3b
34 changed files with 1129 additions and 639 deletions
@@ -1,3 +1,6 @@
+pub mod datablockstream;
+pub mod datastream;
+
 use crate::{EventsItem, PlainEvents, ScalarPlainEvents};
 use async_channel::{Receiver, Sender};
 use err::Error;
@@ -5,7 +8,7 @@ use futures_core::Future;
 use futures_util::StreamExt;
 use items::eventvalues::EventValues;
 use netpod::timeunits::SEC;
-use netpod::{log::*, ChannelArchiver, FilePos, Nanos};
+use netpod::{log::*, ChannelArchiver, DataHeaderPos, FilePos, Nanos};
 use std::convert::TryInto;
 use std::io::{self, SeekFrom};
 use std::path::PathBuf;
@@ -224,6 +227,21 @@ fn readf64(buf: &[u8], pos: usize) -> f64 {
    f64::from_be_bytes(buf.as_ref()[pos..pos + 8].try_into().unwrap())
 }

+fn read_string(buf: &[u8]) -> Result<String, Error> {
+    let imax = buf
+        .iter()
+        .map(|k| *k)
+        .enumerate()
+        .take_while(|&(i, k)| k != 0)
+        .last()
+        .map(|(i, _)| i);
+    let ret = match imax {
+        Some(imax) => String::from_utf8(buf[..imax + 1].to_vec())?,
+        None => String::new(),
+    };
+    Ok(ret)
+}
+
 pub async fn read_file_basics(file: &mut File) -> Result<IndexFileBasics, Error> {
    let mut buf = vec![0; 128];
    read_exact(file, &mut buf[0..4]).await?;
@@ -331,8 +349,8 @@ pub async fn read_file_basics(file: &mut File) -> Result<IndexFileBasics, Error>

 #[derive(Debug)]
 pub struct RTreeNodeRecord {
-    ts1: u64,
-    ts2: u64,
+    ts1: Nanos,
+    ts2: Nanos,
    // TODO should probably be better name `child or offset` and be made enum.
    child_or_id: Offset,
 }
@@ -351,6 +369,12 @@ pub struct RTreeNodeAtRecord {
    rix: usize,
 }

+impl RTreeNodeAtRecord {
+    pub fn rec(&self) -> &RTreeNodeRecord {
+        &self.node.records[self.rix]
+    }
+}
+
 // TODO refactor as struct, rtree_m is a property of the tree.
 pub async fn read_rtree_node(file: &mut File, pos: FilePos, rtree_m: usize) -> Result<RTreeNode, Error> {
    const OFF1: usize = 9;
@@ -388,7 +412,11 @@ pub async fn read_rtree_node(file: &mut File, pos: FilePos, rtree_m: usize) -> R
            let child_or_id = readu64(b, off2 + 16);
            //info!("NODE   {} {}   {} {}   {}", ts1a, ts1b, ts2a, ts2b, child_or_id);
            if child_or_id != 0 {
-                let rec = RTreeNodeRecord { ts1, ts2, child_or_id };
+                let rec = RTreeNodeRecord {
+                    ts1: Nanos { ns: ts1 },
+                    ts2: Nanos { ns: ts2 },
+                    child_or_id,
+                };
                Some(rec)
            } else {
                None
@@ -451,7 +479,7 @@ pub async fn search_record(
        let nr = node.records.len();
        for (i, rec) in node.records.iter().enumerate() {
            //info!("looking at record  i {}", i);
-            if rec.ts2 > beg.ns {
+            if rec.ts2.ns > beg.ns {
                if node.is_leaf {
                    info!("found leaf match at {} / {}", i, nr);
                    let ret = RTreeNodeAtRecord { node, rix: i };
@@ -676,12 +704,22 @@ pub fn datarange_stream(channel_name: &str) -> Result<Receiver<Datarange>, Error
 #[derive(Debug)]
 pub struct Datablock {
    next: Offset,
-    data: Offset,
+    data_header_pos: Offset,
    fname: String,
 }

+impl Datablock {
+    fn file_name(&self) -> &str {
+        &self.fname
+    }
+
+    fn data_header_pos(&self) -> DataHeaderPos {
+        DataHeaderPos(self.data_header_pos)
+    }
+}
+
 async fn read_index_datablockref(file: &mut File, pos: FilePos) -> Result<Datablock, Error> {
-    seek(file, SeekFrom::Start(pos.into())).await?;
+    seek(file, SeekFrom::Start(pos.pos)).await?;
    let mut rb = RingBuf::new();
    rb.fill_min(file, 18).await?;
    let buf = rb.data();
@@ -691,7 +729,11 @@ async fn read_index_datablockref(file: &mut File, pos: FilePos) -> Result<Databl
    rb.fill_min(file, 18 + len).await?;
    let buf = rb.data();
    let fname = String::from_utf8(buf[18..18 + len].to_vec())?;
-    let ret = Datablock { next, data, fname };
+    let ret = Datablock {
+        next,
+        data_header_pos: data,
+        fname,
+    };
    Ok(ret)
 }

@@ -732,7 +774,11 @@ impl DbrType {

 #[derive(Debug)]
 pub struct DatafileHeader {
+    pos: DataHeaderPos,
    dir_offset: u32,
+    // Should be absolute file position of the next data header
+    // together with `fname_next`.
+    // But unfortunately not always set?
    next_offset: u32,
    prev_offset: u32,
    curr_offset: u32,
@@ -743,17 +789,19 @@ pub struct DatafileHeader {
    dbr_type: DbrType,
    dbr_count: usize,
    period: f64,
-    ts1: u64,
-    ts2: u64,
-    ts3: u64,
+    ts_beg: Nanos,
+    ts_end: Nanos,
+    ts_next_file: Nanos,
+    fname_next: String,
+    fname_prev: String,
 }

 const DATA_HEADER_LEN_ON_DISK: usize = 72 + 40 + 40;

-async fn read_datafile_header(file: &mut File, pos: FilePos) -> Result<DatafileHeader, Error> {
-    seek(file, SeekFrom::Start(pos.into())).await?;
+async fn read_datafile_header(file: &mut File, pos: DataHeaderPos) -> Result<DatafileHeader, Error> {
+    seek(file, SeekFrom::Start(pos.0)).await?;
    let mut rb = RingBuf::new();
-    rb.fill_min(file, 88).await?;
+    rb.fill_min(file, DATA_HEADER_LEN_ON_DISK).await?;
    let buf = rb.data();
    let dir_offset = readu32(buf, 0);
    let next_offset = readu32(buf, 4);
@@ -765,7 +813,7 @@ async fn read_datafile_header(file: &mut File, pos: FilePos) -> Result<DatafileH
    let buf_free = readu32(buf, 28);
    let dbr_type = DbrType::from_u16(readu16(buf, 32))?;
    let dbr_count = readu16(buf, 34);
-    let _unused = readu32(buf, 36);
+    // 4 bytes padding.
    let period = readf64(buf, 40);
    let ts1a = readu32(buf, 48);
    let ts1b = readu32(buf, 52);
@@ -773,12 +821,25 @@ async fn read_datafile_header(file: &mut File, pos: FilePos) -> Result<DatafileH
    let ts2b = readu32(buf, 60);
    let ts3a = readu32(buf, 64);
    let ts3b = readu32(buf, 68);
-    let ts1 = ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET;
-    let ts2 = ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET;
-    let ts3 = ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET;
-    // 40 bytes prev-filename.
-    // 40 bytes next-filename.
+    let ts_beg = if ts1a != 0 || ts1b != 0 {
+        ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET
+    } else {
+        0
+    };
+    let ts_end = if ts3a != 0 || ts3b != 0 {
+        ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET
+    } else {
+        0
+    };
+    let ts_next_file = if ts2a != 0 || ts2b != 0 {
+        ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET
+    } else {
+        0
+    };
+    let fname_prev = read_string(&buf[72..112])?;
+    let fname_next = read_string(&buf[112..152])?;
    let ret = DatafileHeader {
+        pos,
        dir_offset,
        next_offset,
        prev_offset,
@@ -790,17 +851,18 @@ async fn read_datafile_header(file: &mut File, pos: FilePos) -> Result<DatafileH
        dbr_type,
        dbr_count: dbr_count as usize,
        period,
-        ts1,
-        ts2,
-        ts3,
+        ts_beg: Nanos { ns: ts_beg },
+        ts_end: Nanos { ns: ts_end },
+        ts_next_file: Nanos { ns: ts_next_file },
+        fname_next,
+        fname_prev,
    };
    Ok(ret)
 }

-async fn read_data_1(file: &mut File, pos: FilePos) -> Result<EventsItem, Error> {
-    let datafile_header = read_datafile_header(file, pos).await?;
-    //info!("datafile_header {:?}", datafile_header);
-    seek(file, SeekFrom::Start(u64::from(pos) + DATA_HEADER_LEN_ON_DISK as u64)).await?;
+async fn read_data_1(file: &mut File, datafile_header: &DatafileHeader) -> Result<EventsItem, Error> {
+    let dhpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64;
+    seek(file, SeekFrom::Start(dhpos)).await?;
    let res = match &datafile_header.dbr_type {
        DbrType::DbrTimeDouble => {
            if datafile_header.dbr_count == 1 {
@@ -946,8 +1008,8 @@ mod test {
    //use disk::{eventblobs::EventChunkerMultifile, eventchunker::EventChunkerConf};

    use super::search_record;
-    use crate::archeng::EPICS_EPOCH_OFFSET;
    use crate::archeng::{open_read, read_channel, read_data_1, read_file_basics, read_index_datablockref};
+    use crate::archeng::{read_datafile_header, EPICS_EPOCH_OFFSET};
    use err::Error;
    use netpod::log::*;
    use netpod::timeunits::*;
@@ -1019,12 +1081,12 @@ mod test {
            assert_eq!(res.node.pos.pos, 8216);
            assert_eq!(res.rix, 17);
            let rec = &res.node.records[res.rix];
-            assert_eq!(rec.ts1, 970351499684884156 + EPICS_EPOCH_OFFSET);
-            assert_eq!(rec.ts2, 970417919634086480 + EPICS_EPOCH_OFFSET);
+            assert_eq!(rec.ts1.ns, 970351499684884156 + EPICS_EPOCH_OFFSET);
+            assert_eq!(rec.ts2.ns, 970417919634086480 + EPICS_EPOCH_OFFSET);
            assert_eq!(rec.child_or_id, 185074);
            let pos = FilePos { pos: rec.child_or_id };
            let datablock = read_index_datablockref(&mut index_file, pos).await?;
-            assert_eq!(datablock.data, 52787);
+            assert_eq!(datablock.data_header_pos, 52787);
            assert_eq!(datablock.fname, "20201001/20201001");
            // The actual datafile for that time was not retained any longer.
            // But the index still points to that.
@@ -1050,17 +1112,17 @@ mod test {
            assert_eq!(res.node.pos.pos, 1861178);
            assert_eq!(res.rix, 41);
            let rec = &res.node.records[res.rix];
-            assert_eq!(rec.ts1, 1001993759871202919 + EPICS_EPOCH_OFFSET);
-            assert_eq!(rec.ts2, 1002009299596362122 + EPICS_EPOCH_OFFSET);
+            assert_eq!(rec.ts1.ns, 1001993759871202919 + EPICS_EPOCH_OFFSET);
+            assert_eq!(rec.ts2.ns, 1002009299596362122 + EPICS_EPOCH_OFFSET);
            assert_eq!(rec.child_or_id, 2501903);
            let pos = FilePos { pos: rec.child_or_id };
            let datablock = read_index_datablockref(&mut index_file, pos).await?;
-            assert_eq!(datablock.data, 9311367);
-            assert_eq!(datablock.fname, "20211001/20211001");
-            let data_path = index_path.parent().unwrap().join(datablock.fname);
+            assert_eq!(datablock.data_header_pos().0, 9311367);
+            assert_eq!(datablock.file_name(), "20211001/20211001");
+            let data_path = index_path.parent().unwrap().join(datablock.file_name());
            let mut data_file = open_read(data_path).await?;
-            let data_pos = FilePos { pos: datablock.data };
-            let events = read_data_1(&mut data_file, data_pos).await?;
+            let datafile_header = read_datafile_header(&mut data_file, datablock.data_header_pos()).await?;
+            let events = read_data_1(&mut data_file, &datafile_header).await?;
            info!("read events: {:?}", events);
            Ok(())
        };
@@ -0,0 +1,310 @@
+use crate::archeng::{
+    open_read, read_channel, read_data_1, read_datafile_header, read_index_datablockref, search_record,
+};
+use crate::EventsItem;
+use async_channel::{Receiver, Sender};
+use err::Error;
+use futures_core::{Future, Stream};
+use futures_util::{FutureExt, StreamExt};
+use items::{RangeCompletableItem, Sitemty, StreamItem, WithLen};
+use netpod::{log::*, DataHeaderPos, FilePos, Nanos};
+use netpod::{Channel, NanoRange};
+use std::collections::VecDeque;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+type FR = (Option<Sitemty<EventsItem>>, Box<dyn FretCb>);
+
+trait FretCb {
+    fn call(&mut self, stream: &mut Pin<&mut DatablockStream>);
+}
+
+async fn datablock_stream(
+    range: NanoRange,
+    channel: Channel,
+    base_dirs: VecDeque<PathBuf>,
+    expand: bool,
+    tx: Sender<Sitemty<EventsItem>>,
+) {
+    match datablock_stream_inner(range, channel, base_dirs, expand, tx.clone()).await {
+        Ok(_) => {}
+        Err(e) => match tx.send(Err(e)).await {
+            Ok(_) => {}
+            Err(e) => {
+                error!("can not forward error: {:?}", e);
+            }
+        },
+    }
+}
+
+async fn datablock_stream_inner(
+    range: NanoRange,
+    channel: Channel,
+    base_dirs: VecDeque<PathBuf>,
+    expand: bool,
+    tx: Sender<Sitemty<EventsItem>>,
+) -> Result<(), Error> {
+    let basename = channel
+        .name()
+        .split("-")
+        .next()
+        .ok_or(Error::with_msg_no_trace("can not find base for channel"))?;
+    for base in base_dirs {
+        debug!(
+            "search for {:?} with basename: {}  in path {:?}",
+            channel, basename, base
+        );
+        let index_path = base.join(format!("archive_{}_SH", basename)).join("index");
+        let res = open_read(index_path.clone()).await;
+        debug!("tried to open index file: {:?}", res);
+        if let Ok(mut index_file) = res {
+            if let Some(basics) = read_channel(&mut index_file, channel.name()).await? {
+                let beg = Nanos { ns: range.beg };
+                let mut search_ts = beg.clone();
+                let mut last_data_file_path = PathBuf::new();
+                let mut last_data_file_pos = DataHeaderPos(0);
+                loop {
+                    // TODO for expand mode, this needs another search function.
+                    let (res, _stats) =
+                        search_record(&mut index_file, basics.rtree_m, basics.rtree_start_pos, search_ts).await?;
+                    if let Some(nrec) = res {
+                        let rec = nrec.rec();
+                        info!("found record: {:?}", rec);
+                        let pos = FilePos { pos: rec.child_or_id };
+                        // TODO rename Datablock?  → IndexNodeDatablock
+                        info!("\n\nREAD Datablock FROM {:?}\n", pos);
+                        let datablock = read_index_datablockref(&mut index_file, pos).await?;
+                        info!("\nDatablock: {:?}\n", datablock);
+                        let data_path = index_path.parent().unwrap().join(datablock.file_name());
+                        if data_path == last_data_file_path && datablock.data_header_pos() == last_data_file_pos {
+                            warn!("SKIP BECAUSE ITS THE SAME BLOCK");
+                        } else {
+                            info!("try to open data_path: {:?}", data_path);
+                            let mut data_file = open_read(data_path.clone()).await?;
+                            let datafile_header =
+                                read_datafile_header(&mut data_file, datablock.data_header_pos()).await?;
+                            info!("datafile_header --------------  HEADER\n{:?}", datafile_header);
+                            let events = read_data_1(&mut data_file, &datafile_header).await?;
+                            info!("Was able to read data: {} events", events.len());
+                            let item = Ok(StreamItem::DataItem(RangeCompletableItem::Data(events)));
+                            tx.send(item).await?;
+                        }
+                        if datablock.next != 0 {
+                            error!("datablock.next != 0:  {:?}", datablock);
+                        }
+                        last_data_file_path = data_path;
+                        last_data_file_pos = datablock.data_header_pos();
+                        if expand {
+                            err::todo()
+                        } else {
+                            search_ts.ns = rec.ts2.ns;
+                        };
+                    } else {
+                        warn!("nothing found, break");
+                        break;
+                    }
+                }
+            }
+        } else {
+            info!("can not find index file at {:?}", index_path);
+        }
+    }
+    Ok(())
+}
+
+pub struct DatablockStream {
+    range: NanoRange,
+    channel: Channel,
+    base_dirs: VecDeque<PathBuf>,
+    expand: bool,
+    fut: Pin<Box<dyn Future<Output = FR>>>,
+    rx: Receiver<Sitemty<EventsItem>>,
+    done: bool,
+    complete: bool,
+}
+
+impl DatablockStream {
+    pub fn for_channel_range(range: NanoRange, channel: Channel, base_dirs: VecDeque<PathBuf>, expand: bool) -> Self {
+        let (tx, rx) = async_channel::bounded(1);
+        taskrun::spawn(datablock_stream(
+            range.clone(),
+            channel.clone(),
+            base_dirs.clone(),
+            expand.clone(),
+            tx,
+        ));
+        let ret = Self {
+            range,
+            channel,
+            base_dirs: VecDeque::new(),
+            expand,
+            fut: Box::pin(Self::start()),
+            rx,
+            done: false,
+            complete: false,
+        };
+        ret
+    }
+
+    async fn start() -> FR {
+        struct Cb {}
+        impl FretCb for Cb {
+            fn call(&mut self, stream: &mut Pin<&mut DatablockStream>) {
+                if let Some(path) = stream.base_dirs.pop_front() {
+                    stream.fut = Box::pin(DatablockStream::start_with_base_dir(path));
+                } else {
+                    // TODO: done?
+                    err::todo();
+                }
+            }
+        }
+        (None, Box::new(Cb {}))
+    }
+
+    async fn start_with_base_dir(path: PathBuf) -> FR {
+        warn!("start_with_base_dir");
+        struct Cb {}
+        impl FretCb for Cb {
+            fn call(&mut self, stream: &mut Pin<&mut DatablockStream>) {
+                let _ = stream;
+            }
+        }
+        (None, Box::new(Cb {}))
+    }
+}
+
+/*
+Loop through all configured data directories.
+Locate the index file.
+Search for the correct Datablock to start with.
+Iterate from there.
+*/
+
+impl Stream for DatablockStream {
+    type Item = Sitemty<EventsItem>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        use Poll::*;
+        loop {
+            break if self.complete {
+                panic!("poll on complete")
+            } else if self.done {
+                self.complete = true;
+                Ready(None)
+            } else if true {
+                self.rx.poll_next_unpin(cx)
+            } else {
+                match self.fut.poll_unpin(cx) {
+                    Ready((k, mut fr)) => {
+                        fr.call(&mut self);
+                        match k {
+                            Some(item) => Ready(Some(item)),
+                            None => continue,
+                        }
+                    }
+                    Pending => Pending,
+                }
+            };
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::EventsItem;
+
+    use super::DatablockStream;
+    use chrono::{DateTime, Utc};
+    use err::Error;
+    use futures_util::StreamExt;
+    use items::{LogItem, Sitemty, StatsItem, StreamItem};
+    use netpod::timeunits::SEC;
+    use netpod::{log::*, RangeFilterStats};
+    use netpod::{Channel, NanoRange};
+    use serde::Serialize;
+    use std::collections::VecDeque;
+    use std::path::PathBuf;
+    use streams::rangefilter::RangeFilter;
+
+    #[test]
+    fn read_file_basic_info() -> Result<(), Error> {
+        let fut = async {
+            // file begin archive_X05DA_SH/20211001/20211001: 1633039259
+            // 1633145759
+            // October 1st CEST: 1633039200
+            // archive_X05DA_SH/20210901/20210920  (has no next-links)
+            // maybe there is no linking across files?
+            // now in this case, there is a `next`. Does the rtree also contain an entry for that?
+            let beg = "2021-10-01T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
+            let end = "2021-10-01T02:00:00Z".parse::<DateTime<Utc>>().unwrap();
+            let range = NanoRange {
+                beg: beg.timestamp() as u64 * SEC,
+                end: end.timestamp() as u64 * SEC,
+            };
+            let channel = Channel {
+                backend: "test-archapp".into(),
+                name: "X05DA-FE-WI1:TC1".into(),
+            };
+            let base_dirs: VecDeque<_> = ["/data/daqbuffer-testdata/sls/gfa03/bl_arch"]
+                .iter()
+                .map(PathBuf::from)
+                .collect();
+            let expand = false;
+            let datablocks = DatablockStream::for_channel_range(range.clone(), channel, base_dirs, expand);
+            let filtered = RangeFilter::<_, EventsItem>::new(datablocks, range, expand);
+            let mut stream = filtered;
+            while let Some(block) = stream.next().await {
+                info!("DatablockStream yields: {:?}", block);
+            }
+            Ok(())
+        };
+        Ok(taskrun::run(fut).unwrap())
+    }
+
+    #[test]
+    fn test_bincode_rep_stats() {
+        fn make_stats<T>() -> Vec<u8>
+        where
+            T: Serialize,
+        {
+            let stats = RangeFilterStats {
+                events_pre: 626262,
+                events_post: 929292,
+                events_unordered: 131313,
+            };
+            let item = StreamItem::Stats(StatsItem::RangeFilterStats(stats));
+            let item: Sitemty<T> = Ok(item);
+            bincode::serialize(&item).unwrap()
+        }
+        let v1 = make_stats::<u8>();
+        let v2 = make_stats::<f32>();
+        let v3 = make_stats::<Vec<u32>>();
+        let v4 = make_stats::<Vec<f64>>();
+        assert_eq!(v1, v2);
+        assert_eq!(v1, v3);
+        assert_eq!(v1, v4);
+    }
+
+    #[test]
+    fn test_bincode_rep_log() {
+        fn make_log<T>() -> Vec<u8>
+        where
+            T: Serialize,
+        {
+            let item = StreamItem::Log(LogItem::quick(
+                Level::DEBUG,
+                format!("Some easy log message for testing purpose here."),
+            ));
+            let item: Sitemty<T> = Ok(item);
+            bincode::serialize(&item).unwrap()
+        }
+        let v1 = make_log::<u8>();
+        let v2 = make_log::<f32>();
+        let v3 = make_log::<Vec<u32>>();
+        let v4 = make_log::<Vec<f64>>();
+        assert_eq!(v1, v2);
+        assert_eq!(v1, v3);
+        assert_eq!(v1, v4);
+    }
+}
@@ -0,0 +1,15 @@
+use crate::EventsItem;
+use futures_core::Stream;
+use items::Sitemty;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+pub struct DataStream {}
+
+impl Stream for DataStream {
+    type Item = Sitemty<EventsItem>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        todo!()
+    }
+}
@@ -239,6 +239,7 @@ macro_rules! arm2 {
                                }
                                _ => panic!(),
                            },
+
                            _ => panic!(),
                        },
                        _ => err::todoval(),
@@ -14,7 +14,7 @@ use items::numops::NumOps;
 use items::waveevents::{WaveEvents, WaveXBinner};
 use items::xbinnedscalarevents::XBinnedScalarEvents;
 use items::xbinnedwaveevents::XBinnedWaveEvents;
-use items::{EventsNodeProcessor, SitemtyFrameType, WithLen, WithTimestamps};
+use items::{Appendable, Clearable, EventsNodeProcessor, PushableIndex, SitemtyFrameType, WithLen, WithTimestamps};
 use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape};
 #[cfg(not(feature = "devread"))]
 pub use parsestub as parse;
@@ -70,6 +70,82 @@ impl ScalarPlainEvents {
    }
 }

+impl Clearable for ScalarPlainEvents {
+    fn clear(&mut self) {
+        match self {
+            ScalarPlainEvents::Byte(k) => k.clear(),
+            ScalarPlainEvents::Short(k) => k.clear(),
+            ScalarPlainEvents::Int(k) => k.clear(),
+            ScalarPlainEvents::Float(k) => k.clear(),
+            ScalarPlainEvents::Double(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for ScalarPlainEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Byte(k) => Self::Byte(k.empty_like_self()),
+            Self::Short(k) => Self::Short(k.empty_like_self()),
+            Self::Int(k) => Self::Int(k.empty_like_self()),
+            Self::Float(k) => Self::Float(k.empty_like_self()),
+            Self::Double(k) => Self::Double(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for ScalarPlainEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for ScalarPlainEvents {
    fn len(&self) -> usize {
        use ScalarPlainEvents::*;
@@ -185,6 +261,82 @@ impl WavePlainEvents {
    }
 }

+impl Clearable for WavePlainEvents {
+    fn clear(&mut self) {
+        match self {
+            WavePlainEvents::Byte(k) => k.clear(),
+            WavePlainEvents::Short(k) => k.clear(),
+            WavePlainEvents::Int(k) => k.clear(),
+            WavePlainEvents::Float(k) => k.clear(),
+            WavePlainEvents::Double(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for WavePlainEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Byte(k) => Self::Byte(k.empty_like_self()),
+            Self::Short(k) => Self::Short(k.empty_like_self()),
+            Self::Int(k) => Self::Int(k.empty_like_self()),
+            Self::Float(k) => Self::Float(k.empty_like_self()),
+            Self::Double(k) => Self::Double(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for WavePlainEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for WavePlainEvents {
    fn len(&self) -> usize {
        use WavePlainEvents::*;
@@ -273,6 +425,82 @@ impl MultiBinWaveEvents {
    }
 }

+impl Clearable for MultiBinWaveEvents {
+    fn clear(&mut self) {
+        match self {
+            MultiBinWaveEvents::Byte(k) => k.clear(),
+            MultiBinWaveEvents::Short(k) => k.clear(),
+            MultiBinWaveEvents::Int(k) => k.clear(),
+            MultiBinWaveEvents::Float(k) => k.clear(),
+            MultiBinWaveEvents::Double(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for MultiBinWaveEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Byte(k) => Self::Byte(k.empty_like_self()),
+            Self::Short(k) => Self::Short(k.empty_like_self()),
+            Self::Int(k) => Self::Int(k.empty_like_self()),
+            Self::Float(k) => Self::Float(k.empty_like_self()),
+            Self::Double(k) => Self::Double(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for MultiBinWaveEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for MultiBinWaveEvents {
    fn len(&self) -> usize {
        use MultiBinWaveEvents::*;
@@ -361,6 +589,82 @@ impl SingleBinWaveEvents {
    }
 }

+impl Clearable for SingleBinWaveEvents {
+    fn clear(&mut self) {
+        match self {
+            SingleBinWaveEvents::Byte(k) => k.clear(),
+            SingleBinWaveEvents::Short(k) => k.clear(),
+            SingleBinWaveEvents::Int(k) => k.clear(),
+            SingleBinWaveEvents::Float(k) => k.clear(),
+            SingleBinWaveEvents::Double(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for SingleBinWaveEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Byte(k) => Self::Byte(k.empty_like_self()),
+            Self::Short(k) => Self::Short(k.empty_like_self()),
+            Self::Int(k) => Self::Int(k.empty_like_self()),
+            Self::Float(k) => Self::Float(k.empty_like_self()),
+            Self::Double(k) => Self::Double(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for SingleBinWaveEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Byte(k) => match src {
+                Self::Byte(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Short(k) => match src {
+                Self::Short(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Int(k) => match src {
+                Self::Int(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Float(k) => match src {
+                Self::Float(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Double(k) => match src {
+                Self::Double(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for SingleBinWaveEvents {
    fn len(&self) -> usize {
        use SingleBinWaveEvents::*;
@@ -440,6 +744,62 @@ impl XBinnedEvents {
    }
 }

+impl Clearable for XBinnedEvents {
+    fn clear(&mut self) {
+        match self {
+            XBinnedEvents::Scalar(k) => k.clear(),
+            XBinnedEvents::SingleBinWave(k) => k.clear(),
+            XBinnedEvents::MultiBinWave(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for XBinnedEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Scalar(k) => Self::Scalar(k.empty_like_self()),
+            Self::SingleBinWave(k) => Self::SingleBinWave(k.empty_like_self()),
+            Self::MultiBinWave(k) => Self::MultiBinWave(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Scalar(k) => match src {
+                Self::Scalar(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::SingleBinWave(k) => match src {
+                Self::SingleBinWave(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::MultiBinWave(k) => match src {
+                Self::MultiBinWave(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for XBinnedEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Scalar(k) => match src {
+                Self::Scalar(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::SingleBinWave(k) => match src {
+                Self::SingleBinWave(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::MultiBinWave(k) => match src {
+                Self::MultiBinWave(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for XBinnedEvents {
    fn len(&self) -> usize {
        use XBinnedEvents::*;
@@ -516,6 +876,52 @@ impl PlainEvents {
    }
 }

+impl Clearable for PlainEvents {
+    fn clear(&mut self) {
+        match self {
+            PlainEvents::Scalar(k) => k.clear(),
+            PlainEvents::Wave(k) => k.clear(),
+        }
+    }
+}
+
+impl Appendable for PlainEvents {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            Self::Scalar(k) => Self::Scalar(k.empty_like_self()),
+            Self::Wave(k) => Self::Wave(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            PlainEvents::Scalar(k) => match src {
+                Self::Scalar(j) => k.append(j),
+                _ => panic!(),
+            },
+            PlainEvents::Wave(k) => match src {
+                Self::Wave(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for PlainEvents {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Scalar(k) => match src {
+                Self::Scalar(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::Wave(k) => match src {
+                Self::Wave(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
 impl WithLen for PlainEvents {
    fn len(&self) -> usize {
        use PlainEvents::*;
@@ -585,7 +991,7 @@ impl EventsItem {
        }
    }

-    pub fn x_aggregate(self, ak: &AggKind) -> EventsItem {
+    pub fn x_aggregate(self, ak: &AggKind) -> Self {
        use EventsItem::*;
        match self {
            Plain(k) => k.x_aggregate(ak),
@@ -614,6 +1020,52 @@ impl WithTimestamps for EventsItem {
    }
 }

+impl Appendable for EventsItem {
+    fn empty_like_self(&self) -> Self {
+        match self {
+            EventsItem::Plain(k) => EventsItem::Plain(k.empty_like_self()),
+            EventsItem::XBinnedEvents(k) => EventsItem::XBinnedEvents(k.empty_like_self()),
+        }
+    }
+
+    fn append(&mut self, src: &Self) {
+        match self {
+            Self::Plain(k) => match src {
+                Self::Plain(j) => k.append(j),
+                _ => panic!(),
+            },
+            Self::XBinnedEvents(k) => match src {
+                Self::XBinnedEvents(j) => k.append(j),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl PushableIndex for EventsItem {
+    fn push_index(&mut self, src: &Self, ix: usize) {
+        match self {
+            Self::Plain(k) => match src {
+                Self::Plain(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+            Self::XBinnedEvents(k) => match src {
+                Self::XBinnedEvents(j) => k.push_index(j, ix),
+                _ => panic!(),
+            },
+        }
+    }
+}
+
+impl Clearable for EventsItem {
+    fn clear(&mut self) {
+        match self {
+            EventsItem::Plain(k) => k.clear(),
+            EventsItem::XBinnedEvents(k) => k.clear(),
+        }
+    }
+}
+
 impl HasShape for EventsItem {
    fn shape(&self) -> Shape {
        use EventsItem::*;