daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on time weight bins

Browse Source
This commit is contained in:
Dominik Werder
2021-09-08 13:45:46 +02:00
parent 0a05082da8
commit 75abb4140a
20 changed files with 357 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
archapp/src/events.rs
View File

@@ -9,12 +9,10 @@ use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::eventvalues::EventValues;
use items::waveevents::{WaveEvents, WaveXBinner};
use items::waveevents::WaveEvents;
use items::xbinnedscalarevents::XBinnedScalarEvents;
use items::xbinnedwaveevents::XBinnedWaveEvents;
use items::{
EventsNodeProcessor, Framable, RangeCompletableItem, Sitemty, SitemtyFrameType, StreamItem, WithLen, WithTimestamps,
};
use items::{Framable, RangeCompletableItem, Sitemty, SitemtyFrameType, StreamItem, WithLen, WithTimestamps};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::timeunits::{DAY, SEC};
@@ -106,6 +104,7 @@ impl StorageMerge {
let mut i1 = self.inprng;
let mut j1 = not_found;
let mut tsmin = u64::MAX;
#[allow(unused)]
use items::{WithLen, WithTimestamps};
loop {
if self.completed_inps[i1] {
@@ -175,7 +174,8 @@ struct FrameMaker {
}
impl FrameMaker {
fn make_frame_gen<T>(item: Sitemty<EventsItem>) -> Box<dyn Framable>
#[allow(dead_code)]
fn make_frame_gen<T>(_item: Sitemty<EventsItem>) -> Box<dyn Framable>
where
T: SitemtyFrameType + Serialize + Send + 'static,
{
@@ -183,6 +183,7 @@ impl FrameMaker {
}
}
#[allow(unused_macros)]
macro_rules! events_item_to_sitemty {
($ei:expr, $t1:ident, $t2:ident, $t3:ident) => {{
let ret = match $ei {
@@ -501,7 +502,7 @@ async fn position_file_for_evq(mut file: File, evq: RawEventsQuery, year: u32) -
}
}
async fn position_file_for_evq_linear(mut file: File, evq: RawEventsQuery, year: u32) -> Result<PositionResult, Error> {
async fn position_file_for_evq_linear(file: File, evq: RawEventsQuery, _year: u32) -> Result<PositionResult, Error> {
let mut pbr = PbFileReader::new(file).await;
pbr.read_header().await?;
loop {

102
archapp/src/lib.rs
View File

@@ -13,7 +13,7 @@ use items::numops::NumOps;
use items::waveevents::{WaveEvents, WaveXBinner};
use items::xbinnedscalarevents::XBinnedScalarEvents;
use items::xbinnedwaveevents::XBinnedWaveEvents;
use items::{EventsNodeProcessor, Framable, SitemtyFrameType, WithLen, WithTimestamps};
use items::{EventsNodeProcessor, SitemtyFrameType, WithLen, WithTimestamps};
use netpod::{AggKind, HasScalarType, HasShape, ScalarType, Shape};
#[cfg(not(feature = "devread"))]
pub use parsestub as parse;
@@ -60,11 +60,11 @@ impl ScalarPlainEvents {
pub fn variant_name(&self) -> String {
use ScalarPlainEvents::*;
match self {
Byte(h) => format!("Byte"),
Short(h) => format!("Short"),
Int(h) => format!("Int"),
Float(h) => format!("Float"),
Double(h) => format!("Double"),
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
}
@@ -97,7 +97,6 @@ impl WithTimestamps for ScalarPlainEvents {
impl HasShape for ScalarPlainEvents {
fn shape(&self) -> Shape {
use ScalarPlainEvents::*;
match self {
_ => Shape::Scalar,
}
@@ -108,11 +107,11 @@ impl HasScalarType for ScalarPlainEvents {
fn scalar_type(&self) -> ScalarType {
use ScalarPlainEvents::*;
match self {
Byte(h) => ScalarType::I8,
Short(h) => ScalarType::I16,
Int(h) => ScalarType::I32,
Float(h) => ScalarType::F32,
Double(h) => ScalarType::F64,
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
@@ -126,13 +125,16 @@ pub enum WavePlainEvents {
Double(WaveEvents<f64>),
}
fn tmp1() {
let ev = EventValues::<u8> {
fn _tmp1() {
let _ev = EventValues::<u8> {
tss: vec![],
values: vec![],
};
<u8 as NumOps>::is_nan(err::todoval());
<EventValues<u8> as SitemtyFrameType>::FRAME_TYPE_ID;
if <EventValues<u8> as SitemtyFrameType>::FRAME_TYPE_ID == 0 {
// Just a dummy..
panic!();
}
//<Vec<u8> as NumOps>::is_nan(err::todoval());
//<EventValues<Vec<u8>> as SitemtyFrameType>::FRAME_TYPE_ID;
}
@@ -224,11 +226,11 @@ impl HasScalarType for WavePlainEvents {
fn scalar_type(&self) -> ScalarType {
use WavePlainEvents::*;
match self {
Byte(h) => ScalarType::I8,
Short(h) => ScalarType::I16,
Int(h) => ScalarType::I32,
Float(h) => ScalarType::F32,
Double(h) => ScalarType::F64,
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
@@ -246,11 +248,11 @@ impl MultiBinWaveEvents {
pub fn variant_name(&self) -> String {
use MultiBinWaveEvents::*;
match self {
Byte(h) => format!("Byte"),
Short(h) => format!("Short"),
Int(h) => format!("Int"),
Float(h) => format!("Float"),
Double(h) => format!("Double"),
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
@@ -298,11 +300,11 @@ impl HasShape for MultiBinWaveEvents {
fn shape(&self) -> Shape {
use MultiBinWaveEvents::*;
match self {
Byte(h) => Shape::Scalar,
Short(h) => Shape::Scalar,
Int(h) => Shape::Scalar,
Float(h) => Shape::Scalar,
Double(h) => Shape::Scalar,
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
}
}
}
@@ -311,11 +313,11 @@ impl HasScalarType for MultiBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use MultiBinWaveEvents::*;
match self {
Byte(h) => ScalarType::I8,
Short(h) => ScalarType::I16,
Int(h) => ScalarType::I32,
Float(h) => ScalarType::F32,
Double(h) => ScalarType::F64,
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}
@@ -333,11 +335,11 @@ impl SingleBinWaveEvents {
pub fn variant_name(&self) -> String {
use SingleBinWaveEvents::*;
match self {
Byte(h) => format!("Byte"),
Short(h) => format!("Short"),
Int(h) => format!("Int"),
Float(h) => format!("Float"),
Double(h) => format!("Double"),
Byte(_) => format!("Byte"),
Short(_) => format!("Short"),
Int(_) => format!("Int"),
Float(_) => format!("Float"),
Double(_) => format!("Double"),
}
}
@@ -385,11 +387,11 @@ impl HasShape for SingleBinWaveEvents {
fn shape(&self) -> Shape {
use SingleBinWaveEvents::*;
match self {
Byte(h) => Shape::Scalar,
Short(h) => Shape::Scalar,
Int(h) => Shape::Scalar,
Float(h) => Shape::Scalar,
Double(h) => Shape::Scalar,
Byte(_) => Shape::Scalar,
Short(_) => Shape::Scalar,
Int(_) => Shape::Scalar,
Float(_) => Shape::Scalar,
Double(_) => Shape::Scalar,
}
}
}
@@ -398,11 +400,11 @@ impl HasScalarType for SingleBinWaveEvents {
fn scalar_type(&self) -> ScalarType {
use SingleBinWaveEvents::*;
match self {
Byte(h) => ScalarType::I8,
Short(h) => ScalarType::I16,
Int(h) => ScalarType::I32,
Float(h) => ScalarType::F32,
Double(h) => ScalarType::F64,
Byte(_) => ScalarType::I8,
Short(_) => ScalarType::I16,
Int(_) => ScalarType::I32,
Float(_) => ScalarType::F32,
Double(_) => ScalarType::F64,
}
}
}

5
archapp/src/parse.rs
View File

@@ -387,7 +387,7 @@ impl LruCache {
tss.sort_unstable();
let thr = tss[1500];
let m1 = std::mem::replace(&mut self.map, BTreeMap::new());
self.map = m1.into_iter().filter(|(j, k)| k > &thr).collect();
self.map = m1.into_iter().filter(|(_j, k)| k > &thr).collect();
}
}
@@ -421,7 +421,6 @@ pub async fn scan_files_inner(
let proots = proot.to_str().unwrap().to_string();
let meta = tokio::fs::metadata(&proot).await?;
let mut paths = VecDeque::new();
let mut waves_found = 0;
paths.push_back(PE {
path: proot,
fty: meta.file_type(),
@@ -451,7 +450,7 @@ pub async fn scan_files_inner(
} else if pe.fty.is_file() {
//tx.send(Ok(Box::new(path.clone()) as RT1)).await?;
let fns = pe.path.to_str().ok_or_else(|| Error::with_msg("invalid path string"))?;
if let Ok(fnp) = parse_data_filename(&fns) {
if let Ok(_fnp) = parse_data_filename(&fns) {
//tx.send(Ok(Box::new(serde_json::to_value(fns)?) as ItemSerBox)).await?;
let channel_path = &fns[proots.len() + 1..fns.len() - 11];
if !lru.query(channel_path) {

2
archapp/src/parse/multi.rs
View File

@@ -37,7 +37,7 @@ pub fn parse_all_ts(off: u64, buf: &[u8], payload_type: PayloadType, year: u32)
ret.push(h);
}
}
Err(e) => {
Err(_e) => {
// TODO ignore except if it's the last chunk.
}
}