WIP on get channel info for arch app

disk/src/agg/binnedt.rs

@@ -1,8 +1,8 @@
-use crate::agg::streams::{Appendable, StreamItem};
-use crate::binned::{FilterFittingInside, RangeCompletableItem, RangeOverlapInfo, ReadableFromFile};
-use crate::Sitemty;
+use crate::agg::streams::Appendable;
+use crate::binned::{FilterFittingInside, RangeOverlapInfo, ReadableFromFile};
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{RangeCompletableItem, Sitemty, StreamItem};
 use netpod::log::*;
 use netpod::{BinnedRange, NanoRange};
 use serde::Serialize;

disk/src/agg/enp.rs

@@ -8,6 +8,7 @@ use crate::binned::{
 };
 use crate::decode::EventValues;
 use err::Error;
+use items::{SitemtyFrameType, SubFrId};
 use netpod::log::*;
 use netpod::timeunits::{MS, SEC};
 use netpod::{x_bin_count, AggKind, NanoRange, Shape};
@@ -44,6 +45,13 @@ pub struct XBinnedScalarEvents<NTY> {
     avgs: Vec<f32>,
 }
 
+impl<NTY> SitemtyFrameType for XBinnedScalarEvents<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0x600 + NTY::SUB;
+}
+
 impl<NTY> XBinnedScalarEvents<NTY> {
     pub fn empty() -> Self {
         Self {
@@ -381,6 +389,13 @@ pub struct XBinnedWaveEvents<NTY> {
     avgs: Vec<Vec<f32>>,
 }
 
+impl<NTY> SitemtyFrameType for XBinnedWaveEvents<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0x900 + NTY::SUB;
+}
+
 impl<NTY> XBinnedWaveEvents<NTY> {
     pub fn empty() -> Self {
         Self {
@@ -719,6 +734,13 @@ pub struct WaveEvents<NTY> {
     pub vals: Vec<Vec<NTY>>,
 }
 
+impl<NTY> SitemtyFrameType for WaveEvents<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0x800 + NTY::SUB;
+}
+
 impl<NTY> WaveEvents<NTY> {
     pub fn empty() -> Self {
         Self {

disk/src/agg/eventbatch.rs

@@ -1,8 +1,9 @@
-use crate::agg::streams::{Appendable, StreamItem};
-use crate::binned::{MakeBytesFrame, RangeCompletableItem, RangeOverlapInfo};
+use crate::agg::streams::Appendable;
+use crate::binned::{MakeBytesFrame, RangeOverlapInfo};
 use crate::frame::makeframe::make_frame;
 use bytes::{BufMut, Bytes, BytesMut};
 use err::Error;
+use items::{RangeCompletableItem, SitemtyFrameType, StreamItem};
 use netpod::log::*;
 use netpod::NanoRange;
 use serde::{Deserialize, Serialize};
@@ -16,6 +17,10 @@ pub struct MinMaxAvgScalarEventBatch {
     pub avgs: Vec<f32>,
 }
 
+impl SitemtyFrameType for MinMaxAvgScalarEventBatch {
+    const FRAME_TYPE_ID: u32 = 0x300;
+}
+
 impl MinMaxAvgScalarEventBatch {
     pub fn empty() -> Self {
         Self {

disk/src/agg/scalarbinbatch.rs

@@ -1,9 +1,10 @@
-use crate::agg::streams::{Appendable, StreamItem, ToJsonBytes};
+use crate::agg::streams::{Appendable, ToJsonBytes};
 use crate::agg::{Fits, FitsInside};
-use crate::binned::{MakeBytesFrame, RangeCompletableItem};
+use crate::binned::MakeBytesFrame;
 use crate::frame::makeframe::make_frame;
 use bytes::{BufMut, Bytes, BytesMut};
 use err::Error;
+use items::{RangeCompletableItem, SitemtyFrameType, StreamItem};
 use netpod::log::*;
 use netpod::timeunits::SEC;
 use netpod::NanoRange;
@@ -21,6 +22,10 @@ pub struct MinMaxAvgScalarBinBatch {
     pub avgs: Vec<f32>,
 }
 
+impl SitemtyFrameType for MinMaxAvgScalarBinBatch {
+    const FRAME_TYPE_ID: u32 = 0x200;
+}
+
 impl MinMaxAvgScalarBinBatch {
     pub fn empty() -> Self {
         Self {

disk/src/agg/streams.rs

@@ -1,20 +1,6 @@
 use crate::binned::WithLen;
-use crate::streamlog::LogItem;
 use err::Error;
-use netpod::EventDataReadStats;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Serialize, Deserialize)]
-pub enum StatsItem {
-    EventDataReadStats(EventDataReadStats),
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub enum StreamItem<T> {
-    DataItem(T),
-    Log(LogItem),
-    Stats(StatsItem),
-}
+use serde::Serialize;
 
 pub trait Collector: Send + Unpin + WithLen {
     type Input: Collectable;

disk/src/binned.rs

@@ -2,23 +2,22 @@ use crate::agg::binnedt::{TBinnerStream, TimeBinnableType, TimeBinnableTypeAggre
 use crate::agg::enp::ts_offs_from_abs;
 use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
 use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
-use crate::agg::streams::{Appendable, Collectable, Collector, StreamItem, ToJsonBytes, ToJsonResult};
+use crate::agg::streams::{Appendable, Collectable, Collector, ToJsonBytes, ToJsonResult};
 use crate::agg::{Fits, FitsInside};
 use crate::binned::binnedfrompbv::BinnedFromPreBinned;
 use crate::binned::query::BinnedQuery;
 use crate::binnedstream::BoxedStream;
 use crate::channelexec::{channel_exec, collect_plain_events_json, ChannelExecFunction};
 use crate::decode::{Endianness, EventValueFromBytes, EventValueShape, EventValues, NumFromBytes};
-use crate::frame::makeframe::{Framable, FrameType, SubFrId};
 use crate::merge::mergedfromremotes::MergedFromRemotes;
-use crate::raw::RawEventsQuery;
-use crate::Sitemty;
 use bytes::Bytes;
 use chrono::{TimeZone, Utc};
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{Framable, FrameType, RangeCompletableItem, Sitemty, SitemtyFrameType, StreamItem, SubFrId};
 use netpod::log::*;
+use netpod::query::RawEventsQuery;
 use netpod::timeunits::SEC;
 use netpod::{
     x_bin_count, AggKind, BinnedRange, BoolNum, NanoRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator,
@@ -686,6 +685,13 @@ pub struct MinMaxAvgBins<NTY> {
     pub avgs: Vec<Option<f32>>,
 }
 
+impl<NTY> SitemtyFrameType for MinMaxAvgBins<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0x700 + NTY::SUB;
+}
+
 impl<NTY> fmt::Debug for MinMaxAvgBins<NTY>
 where
     NTY: fmt::Debug,
@@ -1162,12 +1168,6 @@ where
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub enum RangeCompletableItem<T> {
-    RangeComplete,
-    Data(T),
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct MinMaxAvgWaveBins<NTY> {
     pub ts1s: Vec<u64>,
@@ -1178,6 +1178,13 @@ pub struct MinMaxAvgWaveBins<NTY> {
     pub avgs: Vec<Option<Vec<f32>>>,
 }
 
+impl<NTY> SitemtyFrameType for MinMaxAvgWaveBins<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0xa00 + NTY::SUB;
+}
+
 impl<NTY> fmt::Debug for MinMaxAvgWaveBins<NTY>
 where
     NTY: fmt::Debug,

disk/src/binned/binnedfrompbv.rs

@@ -1,15 +1,13 @@
 use crate::agg::binnedt::{TBinnerStream, TimeBinnableType};
-use crate::agg::streams::StreamItem;
 use crate::binned::query::{CacheUsage, PreBinnedQuery};
-use crate::binned::RangeCompletableItem;
 use crate::cache::{node_ix_for_patch, HttpBodyAsAsyncRead};
 use crate::frame::inmem::InMemoryFrameAsyncReadStream;
-use crate::frame::makeframe::{decode_frame, FrameType};
-use crate::Sitemty;
+use crate::frame::makeframe::decode_frame;
 use err::Error;
 use futures_core::Stream;
 use futures_util::{FutureExt, StreamExt};
 use http::{StatusCode, Uri};
+use items::{FrameType, RangeCompletableItem, Sitemty, StreamItem};
 use netpod::log::*;
 use netpod::{
     x_bin_count, AggKind, AppendToUrl, BinnedRange, ByteSize, Channel, NodeConfigCached, PerfOpts,

disk/src/binned/dim1.rs

@@ -5,9 +5,9 @@ use crate::agg::{Fits, FitsInside};
 use crate::binned::{
     Bool, FilterFittingInside, IsoDateTime, NumOps, RangeOverlapInfo, ReadPbv, ReadableFromFile, TimeBins, WithLen,
 };
-use crate::Sitemty;
 use chrono::{TimeZone, Utc};
 use err::Error;
+use items::{Sitemty, SitemtyFrameType, SubFrId};
 use netpod::log::*;
 use netpod::timeunits::SEC;
 use netpod::NanoRange;
@@ -27,6 +27,13 @@ pub struct MinMaxAvgDim1Bins<NTY> {
     pub avgs: Vec<Option<Vec<f32>>>,
 }
 
+impl<NTY> SitemtyFrameType for MinMaxAvgDim1Bins<NTY>
+where
+    NTY: SubFrId,
+{
+    const FRAME_TYPE_ID: u32 = 0xb00 + NTY::SUB;
+}
+
 impl<NTY> fmt::Debug for MinMaxAvgDim1Bins<NTY>
 where
     NTY: fmt::Debug,

disk/src/binned/pbv.rs

@@ -1,19 +1,18 @@
 use crate::agg::binnedt::{TBinnerStream, TimeBinnableType};
-use crate::agg::streams::{Appendable, StreamItem};
+use crate::agg::streams::Appendable;
 use crate::binned::binnedfrompbv::FetchedPreBinned;
 use crate::binned::query::{CacheUsage, PreBinnedQuery};
-use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, RangeCompletableItem, ReadableFromFile, WithLen};
+use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, ReadableFromFile, WithLen};
 use crate::cache::{write_pb_cache_min_max_avg_scalar, CacheFileDesc, WrittenPbCache};
 use crate::decode::{Endianness, EventValueFromBytes, EventValueShape, NumFromBytes};
-use crate::frame::makeframe::FrameType;
 use crate::merge::mergedfromremotes::MergedFromRemotes;
-use crate::raw::RawEventsQuery;
 use crate::streamlog::Streamlog;
-use crate::Sitemty;
 use err::Error;
 use futures_core::Stream;
 use futures_util::{FutureExt, StreamExt};
+use items::{FrameType, RangeCompletableItem, Sitemty, StreamItem};
 use netpod::log::*;
+use netpod::query::RawEventsQuery;
 use netpod::{
     x_bin_count, AggKind, BinnedRange, NodeConfigCached, PerfOpts, PreBinnedPatchIterator, PreBinnedPatchRange, Shape,
 };

disk/src/binned/prebinned.rs

@@ -8,12 +8,11 @@ use crate::decode::{
     BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValuesDim0Case, EventValuesDim1Case,
     LittleEndian, NumFromBytes,
 };
-use crate::frame::makeframe::{Framable, FrameType};
-use crate::Sitemty;
 use bytes::Bytes;
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{Framable, FrameType, Sitemty};
 use netpod::{AggKind, BoolNum, ByteOrder, NodeConfigCached, ScalarType, Shape};
 use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
 use serde::de::DeserializeOwned;

disk/src/channelexec.rs

@@ -1,21 +1,20 @@
 use crate::agg::binnedt::TimeBinnableType;
 use crate::agg::enp::Identity;
-use crate::agg::streams::{Collectable, Collector, StreamItem};
-use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex, RangeCompletableItem};
+use crate::agg::streams::{Collectable, Collector};
+use crate::binned::{EventsNodeProcessor, NumOps, PushableIndex};
 use crate::decode::{
     BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValues, EventValuesDim0Case, EventValuesDim1Case,
     LittleEndian, NumFromBytes,
 };
-use crate::frame::makeframe::{Framable, FrameType};
 use crate::merge::mergedfromremotes::MergedFromRemotes;
-use crate::raw::RawEventsQuery;
-use crate::Sitemty;
 use bytes::Bytes;
 use err::Error;
 use futures_core::Stream;
 use futures_util::future::FutureExt;
 use futures_util::StreamExt;
+use items::{Framable, FrameType, RangeCompletableItem, Sitemty, StreamItem};
 use netpod::log::*;
+use netpod::query::RawEventsQuery;
 use netpod::{AggKind, BoolNum, ByteOrder, Channel, NanoRange, NodeConfigCached, PerfOpts, ScalarType, Shape};
 use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
 use serde::de::DeserializeOwned;

disk/src/decode.rs

@@ -1,16 +1,17 @@
 use crate::agg::binnedt::TimeBinnableType;
 use crate::agg::enp::{ts_offs_from_abs, Identity, WaveNBinner, WavePlainProc, WaveXBinner};
-use crate::agg::streams::{Appendable, Collectable, Collector, StreamItem};
+use crate::agg::streams::{Appendable, Collectable, Collector};
 use crate::agg::{Fits, FitsInside};
 use crate::binned::{
     Bool, EventValuesAggregator, EventsNodeProcessor, FilterFittingInside, MinMaxAvgBins, NumOps, PushableIndex,
-    RangeCompletableItem, RangeOverlapInfo, ReadPbv, ReadableFromFile, WithLen, WithTimestamps,
+    RangeOverlapInfo, ReadPbv, ReadableFromFile, WithLen, WithTimestamps,
 };
 use crate::eventblobs::EventChunkerMultifile;
 use crate::eventchunker::EventFull;
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{RangeCompletableItem, SitemtyFrameType, StreamItem};
 use netpod::{BoolNum, NanoRange};
 use serde::{Deserialize, Serialize};
 use std::marker::PhantomData;
@@ -190,6 +191,13 @@ pub struct EventValues<VT> {
     pub values: Vec<VT>,
 }
 
+impl<NTY> SitemtyFrameType for EventValues<NTY>
+where
+    NTY: NumOps,
+{
+    const FRAME_TYPE_ID: u32 = 0x500 + NTY::SUB;
+}
+
 impl<VT> EventValues<VT> {
     pub fn empty() -> Self {
         Self {

disk/src/eventblobs.rs

@@ -1,12 +1,10 @@
-use crate::agg::streams::StreamItem;
-use crate::binned::RangeCompletableItem;
 use crate::dataopen::{open_files, OpenedFile};
 use crate::eventchunker::{EventChunker, EventChunkerConf, EventFull};
 use crate::file_content_stream;
-use crate::streamlog::LogItem;
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{LogItem, RangeCompletableItem, StreamItem};
 use netpod::log::*;
 use netpod::timeunits::SEC;
 use netpod::{ChannelConfig, NanoRange, Node};

disk/src/eventchunker.rs

@@ -1,11 +1,10 @@
-use crate::agg::streams::{StatsItem, StreamItem};
-use crate::binned::RangeCompletableItem;
 use crate::{FileChunkRead, NeedMinBuffer};
 use bitshuffle::bitshuffle_decompress;
 use bytes::{Buf, BytesMut};
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{RangeCompletableItem, StatsItem, StreamItem};
 use netpod::log::*;
 use netpod::timeunits::SEC;
 use netpod::{ByteSize, ChannelConfig, EventDataReadStats, NanoRange, ScalarType, Shape};

disk/src/frame/inmem.rs

@@ -1,9 +1,9 @@
-use crate::agg::streams::StreamItem;
-use crate::frame::makeframe::{INMEM_FRAME_FOOT, INMEM_FRAME_HEAD, INMEM_FRAME_MAGIC};
 use bytes::{BufMut, Bytes, BytesMut};
 use err::Error;
 use futures_core::Stream;
 use futures_util::pin_mut;
+use items::StreamItem;
+use items::{INMEM_FRAME_FOOT, INMEM_FRAME_HEAD, INMEM_FRAME_MAGIC};
 use netpod::log::*;
 use std::pin::Pin;
 use std::task::{Context, Poll};

disk/src/frame/makeframe.rs

@@ -1,256 +1,9 @@
-use crate::agg::enp::{WaveEvents, XBinnedScalarEvents, XBinnedWaveEvents};
-use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
-use crate::agg::scalarbinbatch::MinMaxAvgScalarBinBatch;
-use crate::agg::streams::StreamItem;
-use crate::binned::dim1::MinMaxAvgDim1Bins;
-use crate::binned::{MinMaxAvgBins, MinMaxAvgWaveBins, NumOps, RangeCompletableItem};
-use crate::decode::EventValues;
 use crate::frame::inmem::InMemoryFrame;
-use crate::raw::EventQueryJsonStringFrame;
-use crate::Sitemty;
 use bytes::{BufMut, BytesMut};
 use err::Error;
-use netpod::BoolNum;
+use items::{FrameType, INMEM_FRAME_ENCID, INMEM_FRAME_HEAD, INMEM_FRAME_MAGIC};
 use serde::{de::DeserializeOwned, Serialize};
 
-pub const INMEM_FRAME_ENCID: u32 = 0x12121212;
-pub const INMEM_FRAME_HEAD: usize = 20;
-pub const INMEM_FRAME_FOOT: usize = 4;
-pub const INMEM_FRAME_MAGIC: u32 = 0xc6c3b73d;
-
-pub trait SubFrId {
-    const SUB: u32;
-}
-
-impl SubFrId for u8 {
-    const SUB: u32 = 3;
-}
-
-impl SubFrId for u16 {
-    const SUB: u32 = 5;
-}
-
-impl SubFrId for u32 {
-    const SUB: u32 = 8;
-}
-
-impl SubFrId for u64 {
-    const SUB: u32 = 10;
-}
-
-impl SubFrId for i8 {
-    const SUB: u32 = 2;
-}
-
-impl SubFrId for i16 {
-    const SUB: u32 = 4;
-}
-
-impl SubFrId for i32 {
-    const SUB: u32 = 7;
-}
-
-impl SubFrId for i64 {
-    const SUB: u32 = 9;
-}
-
-impl SubFrId for f32 {
-    const SUB: u32 = 11;
-}
-
-impl SubFrId for f64 {
-    const SUB: u32 = 12;
-}
-
-impl SubFrId for BoolNum {
-    const SUB: u32 = 13;
-}
-
-pub trait FrameType {
-    const FRAME_TYPE_ID: u32;
-}
-
-impl FrameType for EventQueryJsonStringFrame {
-    const FRAME_TYPE_ID: u32 = 0x100;
-}
-
-impl FrameType for Sitemty<MinMaxAvgScalarBinBatch> {
-    const FRAME_TYPE_ID: u32 = 0x200;
-}
-
-impl FrameType for Sitemty<MinMaxAvgScalarEventBatch> {
-    const FRAME_TYPE_ID: u32 = 0x300;
-}
-
-impl<NTY> FrameType for Sitemty<EventValues<NTY>>
-where
-    NTY: NumOps,
-{
-    const FRAME_TYPE_ID: u32 = 0x500 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<XBinnedScalarEvents<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0x600 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<MinMaxAvgBins<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0x700 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<WaveEvents<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0x800 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<XBinnedWaveEvents<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0x900 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<MinMaxAvgWaveBins<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0xa00 + NTY::SUB;
-}
-
-impl<NTY> FrameType for Sitemty<MinMaxAvgDim1Bins<NTY>>
-where
-    NTY: SubFrId,
-{
-    const FRAME_TYPE_ID: u32 = 0xb00 + NTY::SUB;
-}
-
-pub trait ProvidesFrameType {
-    fn frame_type_id(&self) -> u32;
-}
-
-pub trait Framable: Send {
-    fn typeid(&self) -> u32;
-    fn make_frame(&self) -> Result<BytesMut, Error>;
-}
-
-impl Framable for Sitemty<serde_json::Value> {
-    fn typeid(&self) -> u32 {
-        EventQueryJsonStringFrame::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        panic!()
-    }
-}
-
-impl Framable for Result<StreamItem<RangeCompletableItem<MinMaxAvgScalarBinBatch>>, Error> {
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl Framable for Result<StreamItem<RangeCompletableItem<MinMaxAvgScalarEventBatch>>, Error> {
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Result<StreamItem<RangeCompletableItem<EventValues<NTY>>>, err::Error>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Result<StreamItem<RangeCompletableItem<XBinnedScalarEvents<NTY>>>, err::Error>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Sitemty<MinMaxAvgBins<NTY>>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Sitemty<WaveEvents<NTY>>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Sitemty<XBinnedWaveEvents<NTY>>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Sitemty<MinMaxAvgWaveBins<NTY>>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
-impl<NTY> Framable for Sitemty<MinMaxAvgDim1Bins<NTY>>
-where
-    NTY: NumOps + Serialize,
-{
-    fn typeid(&self) -> u32 {
-        Self::FRAME_TYPE_ID
-    }
-    fn make_frame(&self) -> Result<BytesMut, Error> {
-        make_frame(self)
-    }
-}
-
 pub fn make_frame<FT>(item: &FT) -> Result<BytesMut, Error>
 where
     FT: FrameType + Serialize,

disk/src/lib.rs

@@ -1,5 +1,3 @@
-use crate::agg::streams::StreamItem;
-use crate::binned::RangeCompletableItem;
 use crate::dataopen::open_files;
 use crate::dtflags::{ARRAY, BIG_ENDIAN, COMPRESSION, SHAPE};
 use bytes::{Bytes, BytesMut};
@@ -477,8 +475,6 @@ pub fn raw_concat_channel_read_stream_timebin(
     }
 }
 
-pub type Sitemty<T> = Result<StreamItem<RangeCompletableItem<T>>, Error>;
-
 pub mod dtflags {
     pub const COMPRESSION: u8 = 0x80;
     pub const ARRAY: u8 = 0x40;

disk/src/merge.rs

@@ -1,10 +1,9 @@
-use crate::agg::streams::{Appendable, StatsItem, StreamItem};
-use crate::binned::{EventsNodeProcessor, PushableIndex, RangeCompletableItem, WithLen, WithTimestamps};
-use crate::streamlog::LogItem;
-use crate::Sitemty;
+use crate::agg::streams::Appendable;
+use crate::binned::{EventsNodeProcessor, PushableIndex, WithLen, WithTimestamps};
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{LogItem, RangeCompletableItem, Sitemty, StatsItem, StreamItem};
 use netpod::log::*;
 use netpod::EventDataReadStats;
 use std::collections::VecDeque;

disk/src/merge/mergedfromremotes.rs

@@ -1,13 +1,13 @@
 use crate::agg::streams::Appendable;
 use crate::binned::{EventsNodeProcessor, PushableIndex};
-use crate::frame::makeframe::FrameType;
 use crate::merge::MergedStream;
-use crate::raw::{x_processed_stream_from_node, RawEventsQuery};
-use crate::Sitemty;
+use crate::raw::client::x_processed_stream_from_node;
 use err::Error;
 use futures_core::Stream;
 use futures_util::{pin_mut, StreamExt};
+use items::{FrameType, Sitemty};
 use netpod::log::*;
+use netpod::query::RawEventsQuery;
 use netpod::{Cluster, PerfOpts};
 use std::future::Future;
 use std::pin::Pin;

disk/src/raw.rs

@@ -1,61 +1,3 @@
-/*!
-Delivers event data.
-
-Delivers event data (not yet time-binned) from local storage and provides client functions
-to request such data from nodes.
-*/
-
-use crate::agg::streams::StreamItem;
-use crate::binned::{EventsNodeProcessor, RangeCompletableItem};
-use crate::frame::inmem::InMemoryFrameAsyncReadStream;
-use crate::frame::makeframe::{make_frame, make_term_frame, FrameType};
-use crate::raw::eventsfromframes::EventsFromFrames;
-use crate::Sitemty;
-use err::Error;
-use futures_core::Stream;
-use netpod::{AggKind, Channel, NanoRange, Node, PerfOpts};
-use serde::{Deserialize, Serialize};
-use std::pin::Pin;
-use tokio::io::AsyncWriteExt;
-use tokio::net::TcpStream;
-
+pub mod client;
 pub mod conn;
 pub mod eventsfromframes;
-
-/**
-Query parameters to request (optionally) X-processed, but not T-processed events.
-*/
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct RawEventsQuery {
-    pub channel: Channel,
-    pub range: NanoRange,
-    pub agg_kind: AggKind,
-    pub disk_io_buffer_size: usize,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct EventQueryJsonStringFrame(String);
-
-pub async fn x_processed_stream_from_node<ENP>(
-    query: RawEventsQuery,
-    perf_opts: PerfOpts,
-    node: Node,
-) -> Result<Pin<Box<dyn Stream<Item = Sitemty<<ENP as EventsNodeProcessor>::Output>> + Send>>, Error>
-where
-    ENP: EventsNodeProcessor,
-    <ENP as EventsNodeProcessor>::Output: Unpin + 'static,
-    Result<StreamItem<RangeCompletableItem<<ENP as EventsNodeProcessor>::Output>>, err::Error>: FrameType,
-{
-    let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
-    let qjs = serde_json::to_string(&query)?;
-    let (netin, mut netout) = net.into_split();
-    let buf = make_frame(&EventQueryJsonStringFrame(qjs))?;
-    netout.write_all(&buf).await?;
-    let buf = make_term_frame();
-    netout.write_all(&buf).await?;
-    netout.flush().await?;
-    netout.forget();
-    let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
-    let items = EventsFromFrames::new(frames);
-    Ok(Box::pin(items))
-}

disk/src/raw/client.rs

@@ -0,0 +1,43 @@
+/*!
+Delivers event data.
+
+Delivers event data (not yet time-binned) from local storage and provides client functions
+to request such data from nodes.
+ */
+
+use crate::binned::EventsNodeProcessor;
+use crate::frame::inmem::InMemoryFrameAsyncReadStream;
+use crate::frame::makeframe::{make_frame, make_term_frame};
+use crate::raw::eventsfromframes::EventsFromFrames;
+use err::Error;
+use futures_core::Stream;
+use items::{FrameType, RangeCompletableItem, Sitemty, StreamItem};
+use netpod::query::RawEventsQuery;
+use netpod::{EventQueryJsonStringFrame, Node, PerfOpts};
+use std::pin::Pin;
+use tokio::io::AsyncWriteExt;
+use tokio::net::TcpStream;
+
+pub async fn x_processed_stream_from_node<ENP>(
+    query: RawEventsQuery,
+    perf_opts: PerfOpts,
+    node: Node,
+) -> Result<Pin<Box<dyn Stream<Item = Sitemty<<ENP as EventsNodeProcessor>::Output>> + Send>>, Error>
+where
+    ENP: EventsNodeProcessor,
+    <ENP as EventsNodeProcessor>::Output: Unpin + 'static,
+    Result<StreamItem<RangeCompletableItem<<ENP as EventsNodeProcessor>::Output>>, err::Error>: FrameType,
+{
+    let net = TcpStream::connect(format!("{}:{}", node.host, node.port_raw)).await?;
+    let qjs = serde_json::to_string(&query)?;
+    let (netin, mut netout) = net.into_split();
+    let buf = make_frame(&EventQueryJsonStringFrame(qjs))?;
+    netout.write_all(&buf).await?;
+    let buf = make_term_frame();
+    netout.write_all(&buf).await?;
+    netout.flush().await?;
+    netout.forget();
+    let frames = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
+    let items = EventsFromFrames::new(frames);
+    Ok(Box::pin(items))
+}

disk/src/raw/conn.rs

@@ -1,100 +1,18 @@
-use crate::agg::eventbatch::MinMaxAvgScalarEventBatch;
-use crate::agg::streams::StreamItem;
-use crate::binned::{EventsNodeProcessor, NumOps, RangeCompletableItem};
+use crate::binned::{EventsNodeProcessor, NumOps};
 use crate::decode::{
     BigEndian, Endianness, EventValueFromBytes, EventValueShape, EventValuesDim0Case, EventValuesDim1Case,
     EventsDecodedStream, LittleEndian, NumFromBytes,
 };
 use crate::eventblobs::EventChunkerMultifile;
 use crate::eventchunker::EventChunkerConf;
-use crate::frame::inmem::InMemoryFrameAsyncReadStream;
-use crate::frame::makeframe::{decode_frame, make_frame, make_term_frame, Framable};
-use crate::raw::{EventQueryJsonStringFrame, RawEventsQuery};
-use crate::Sitemty;
 use err::Error;
 use futures_core::Stream;
 use futures_util::StreamExt;
-use netpod::log::*;
-use netpod::{AggKind, BoolNum, ByteOrder, ByteSize, NodeConfigCached, PerfOpts, ScalarType, Shape};
+use items::{Framable, RangeCompletableItem, Sitemty, StreamItem};
+use netpod::query::RawEventsQuery;
+use netpod::{AggKind, BoolNum, ByteOrder, ByteSize, NodeConfigCached, ScalarType, Shape};
 use parse::channelconfig::{extract_matching_config_entry, read_local_config, MatchingConfigEntry};
-use std::io;
-use std::net::SocketAddr;
 use std::pin::Pin;
-use tokio::io::AsyncWriteExt;
-use tokio::net::tcp::OwnedWriteHalf;
-use tokio::net::TcpStream;
-use tracing::Instrument;
-
-pub async fn events_service(node_config: NodeConfigCached) -> Result<(), Error> {
-    let addr = format!("{}:{}", node_config.node.listen, node_config.node.port_raw);
-    let lis = tokio::net::TcpListener::bind(addr).await?;
-    loop {
-        match lis.accept().await {
-            Ok((stream, addr)) => {
-                taskrun::spawn(events_conn_handler(stream, addr, node_config.clone()));
-            }
-            Err(e) => Err(e)?,
-        }
-    }
-}
-
-async fn events_conn_handler(stream: TcpStream, addr: SocketAddr, node_config: NodeConfigCached) -> Result<(), Error> {
-    //use tracing_futures::Instrument;
-    let span1 = span!(Level::INFO, "raw::raw_conn_handler");
-    let r = events_conn_handler_inner(stream, addr, &node_config)
-        .instrument(span1)
-        .await;
-    match r {
-        Ok(k) => Ok(k),
-        Err(e) => {
-            error!("raw_conn_handler sees error: {:?}", e);
-            Err(e)
-        }
-    }
-}
-
-async fn events_conn_handler_inner(
-    stream: TcpStream,
-    addr: SocketAddr,
-    node_config: &NodeConfigCached,
-) -> Result<(), Error> {
-    match events_conn_handler_inner_try(stream, addr, node_config).await {
-        Ok(_) => (),
-        Err(mut ce) => {
-            error!("events_conn_handler_inner: {:?}", ce.err);
-            if false {
-                let buf = make_frame::<Result<StreamItem<RangeCompletableItem<MinMaxAvgScalarEventBatch>>, Error>>(
-                    &Err(ce.err),
-                )?;
-                match ce.netout.write_all(&buf).await {
-                    Ok(_) => (),
-                    Err(e) => match e.kind() {
-                        io::ErrorKind::BrokenPipe => {}
-                        _ => {
-                            error!("events_conn_handler_inner sees: {:?}", e);
-                            return Err(e)?;
-                        }
-                    },
-                }
-            }
-        }
-    }
-    Ok(())
-}
-
-struct ConnErr {
-    err: Error,
-    netout: OwnedWriteHalf,
-}
-
-impl<E: Into<Error>> From<(E, OwnedWriteHalf)> for ConnErr {
-    fn from((err, netout): (E, OwnedWriteHalf)) -> Self {
-        Self {
-            err: err.into(),
-            netout,
-        }
-    }
-}
 
 fn make_num_pipeline_stream_evs<NTY, END, EVS, ENP>(
     event_value_shape: EVS,
@@ -206,75 +124,38 @@ macro_rules! pipe1 {
     };
 }
 
-async fn events_conn_handler_inner_try(
-    stream: TcpStream,
-    addr: SocketAddr,
+pub async fn make_event_pipe(
+    evq: &RawEventsQuery,
     node_config: &NodeConfigCached,
-) -> Result<(), ConnErr> {
-    let _ = addr;
-    let (netin, mut netout) = stream.into_split();
-    let perf_opts = PerfOpts { inmem_bufcap: 512 };
-    let mut h = InMemoryFrameAsyncReadStream::new(netin, perf_opts.inmem_bufcap);
-    let mut frames = vec![];
-    while let Some(k) = h
-        .next()
-        .instrument(span!(Level::INFO, "raw_conn_handler  INPUT STREAM READ"))
-        .await
-    {
-        match k {
-            Ok(StreamItem::DataItem(item)) => {
-                frames.push(item);
-            }
-            Ok(_) => {}
-            Err(e) => {
-                return Err((e, netout))?;
-            }
-        }
-    }
-    if frames.len() != 1 {
-        error!("missing command frame");
-        return Err((Error::with_msg("missing command frame"), netout))?;
-    }
-    let qitem: EventQueryJsonStringFrame = match decode_frame(&frames[0]) {
-        Ok(k) => k,
-        Err(e) => return Err((e, netout).into()),
-    };
-    let res: Result<RawEventsQuery, _> = serde_json::from_str(&qitem.0);
-    let evq = match res {
-        Ok(k) => k,
-        Err(e) => {
-            error!("json parse error: {:?}", e);
-            return Err((Error::with_msg("json parse error"), netout))?;
-        }
-    };
-    info!("---------------------------------------------------\nevq {:?}", evq);
+) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable>> + Send>>, Error> {
     match dbconn::channel_exists(&evq.channel, &node_config).await {
         Ok(_) => (),
-        Err(e) => return Err((e, netout))?,
+        Err(e) => return Err(e)?,
     }
     let range = &evq.range;
     let channel_config = match read_local_config(&evq.channel, &node_config.node).await {
         Ok(k) => k,
         Err(e) => {
             if e.msg().contains("ErrorKind::NotFound") {
-                return Ok(());
+                let s = futures_util::stream::empty();
+                return Ok(Box::pin(s));
             } else {
-                return Err((e, netout))?;
+                return Err(e)?;
             }
         }
     };
     let entry_res = match extract_matching_config_entry(range, &channel_config) {
         Ok(k) => k,
-        Err(e) => return Err((e, netout))?,
+        Err(e) => return Err(e)?,
     };
     let entry = match entry_res {
-        MatchingConfigEntry::None => return Err((Error::with_msg("no config entry found"), netout))?,
-        MatchingConfigEntry::Multiple => return Err((Error::with_msg("multiple config entries found"), netout))?,
+        MatchingConfigEntry::None => return Err(Error::with_msg("no config entry found"))?,
+        MatchingConfigEntry::Multiple => return Err(Error::with_msg("multiple config entries found"))?,
         MatchingConfigEntry::Entry(entry) => entry,
     };
     let shape = match entry.to_shape() {
         Ok(k) => k,
-        Err(e) => return Err((e, netout))?,
+        Err(e) => return Err(e)?,
     };
     let channel_config = netpod::ChannelConfig {
         channel: evq.channel.clone(),
@@ -296,28 +177,12 @@ async fn events_conn_handler_inner_try(
         event_chunker_conf,
     );
     let shape = entry.to_shape().unwrap();
-    let mut p1 = pipe1!(entry.scalar_type, entry.byte_order, shape, evq.agg_kind, event_blobs);
-    while let Some(item) = p1.next().await {
-        //info!("conn.rs  encode frame typeid {:x}", item.typeid());
-        let item = item.make_frame();
-        match item {
-            Ok(buf) => match netout.write_all(&buf).await {
-                Ok(_) => {}
-                Err(e) => return Err((e, netout))?,
-            },
-            Err(e) => {
-                return Err((e, netout))?;
-            }
-        }
-    }
-    let buf = make_term_frame();
-    match netout.write_all(&buf).await {
-        Ok(_) => (),
-        Err(e) => return Err((e, netout))?,
-    }
-    match netout.flush().await {
-        Ok(_) => (),
-        Err(e) => return Err((e, netout))?,
-    }
-    Ok(())
+    let pipe = pipe1!(
+        entry.scalar_type,
+        entry.byte_order,
+        shape,
+        evq.agg_kind.clone(),
+        event_blobs
+    );
+    Ok(pipe)
 }

disk/src/raw/eventsfromframes.rs

@@ -1,9 +1,8 @@
-use crate::agg::streams::StreamItem;
 use crate::frame::inmem::InMemoryFrameAsyncReadStream;
-use crate::frame::makeframe::{decode_frame, FrameType};
-use crate::Sitemty;
+use crate::frame::makeframe::decode_frame;
 use futures_core::Stream;
 use futures_util::StreamExt;
+use items::{FrameType, Sitemty, StreamItem};
 use netpod::log::*;
 use serde::de::DeserializeOwned;
 use std::marker::PhantomData;

disk/src/streamlog.rs

@@ -1,88 +1,6 @@
+use items::LogItem;
 use netpod::log::*;
-use serde::de::{Error, Visitor};
-use serde::{Deserialize, Serialize};
 use std::collections::VecDeque;
-use std::fmt::Formatter;
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct LogItem {
-    node_ix: u32,
-    #[serde(with = "levelserde")]
-    level: Level,
-    msg: String,
-}
-
-impl LogItem {
-    pub fn quick(level: Level, msg: String) -> Self {
-        Self {
-            level,
-            msg,
-            node_ix: 42,
-        }
-    }
-}
-
-struct VisitLevel;
-
-impl<'de> Visitor<'de> for VisitLevel {
-    type Value = u32;
-
-    fn expecting(&self, fmt: &mut Formatter) -> std::fmt::Result {
-        write!(fmt, "expect u32 Level code")
-    }
-
-    fn visit_u32<E>(self, v: u32) -> Result<Self::Value, E>
-    where
-        E: Error,
-    {
-        Ok(v)
-    }
-}
-
-mod levelserde {
-    use super::Level;
-    use crate::streamlog::VisitLevel;
-    use serde::{Deserializer, Serializer};
-
-    pub fn serialize<S>(t: &Level, s: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let g = match *t {
-            Level::ERROR => 1,
-            Level::WARN => 2,
-            Level::INFO => 3,
-            Level::DEBUG => 4,
-            Level::TRACE => 5,
-        };
-        s.serialize_u32(g)
-    }
-
-    pub fn deserialize<'de, D>(d: D) -> Result<Level, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        match d.deserialize_u32(VisitLevel) {
-            Ok(level) => {
-                let g = if level == 1 {
-                    Level::ERROR
-                } else if level == 2 {
-                    Level::WARN
-                } else if level == 3 {
-                    Level::INFO
-                } else if level == 4 {
-                    Level::DEBUG
-                } else if level == 5 {
-                    Level::TRACE
-                } else {
-                    Level::TRACE
-                };
-                Ok(g)
-            }
-            Err(e) => Err(e),
-        }
-    }
-}
 
 pub struct Streamlog {
     items: VecDeque<LogItem>,