daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add status board for post-fetch response status (for py data api 3)

Browse Source
This commit is contained in:
Dominik Werder
2022-03-02 09:55:11 +01:00
parent aa6db72e8e
commit 0b741d187e
7 changed files with 385 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
disk/src/eventblobs.rs
View File

@@ -10,8 +10,6 @@ use netpod::timeunits::SEC;
use netpod::{log::*, FileIoBufferSize};
use netpod::{ChannelConfig, NanoRange, Node};
use std::pin::Pin;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
use std::task::{Context, Poll};
use streams::rangefilter::RangeFilter;
@@ -29,11 +27,11 @@ pub struct EventChunkerMultifile {
data_completed: bool,
errored: bool,
completed: bool,
max_ts: Arc<AtomicU64>,
files_count: u32,
node_ix: usize,
expand: bool,
do_decompress: bool,
max_ts: u64,
}
impl EventChunkerMultifile {
@@ -62,11 +60,11 @@ impl EventChunkerMultifile {
data_completed: false,
errored: false,
completed: false,
max_ts: Arc::new(AtomicU64::new(0)),
files_count: 0,
node_ix,
expand,
do_decompress,
max_ts: 0,
}
}
}
@@ -81,7 +79,7 @@ impl Stream for EventChunkerMultifile {
use Poll::*;
'outer: loop {
break if self.completed {
panic!("EventBlobsComplete poll_next on completed");
panic!("EventChunkerMultifile poll_next on completed");
} else if self.errored {
self.completed = true;
return Ready(None);
@@ -93,11 +91,35 @@ impl Stream for EventChunkerMultifile {
Some(evs) => match evs.poll_next_unpin(cx) {
Ready(Some(k)) => {
if let Ok(StreamItem::DataItem(RangeCompletableItem::Data(h))) = &k {
if false {
info!("EventChunkerMultifile emit {} events", h.tss.len());
};
if let Some(&g) = h.tss.last() {
if g == self.max_ts {
let msg = format!("EventChunkerMultifile repeated ts {}", g);
error!("{}", msg);
let e = Error::with_msg(msg);
self.errored = true;
Ready(Some(Err(e)))
} else if g < self.max_ts {
let msg = format!("EventChunkerMultifile unordered ts {}", g);
error!("{}", msg);
let e = Error::with_msg(msg);
self.errored = true;
Ready(Some(Err(e)))
} else {
self.max_ts = g;
if true {
info!("EventChunkerMultifile emit {} events", h.tss.len());
}
Ready(Some(k))
}
} else {
Ready(Some(k))
}
} else if let Err(_) = &k {
self.errored = true;
Ready(Some(k))
} else {
Ready(Some(k))
}
Ready(Some(k))
}
Ready(None) => {
self.evs = None;
@@ -128,7 +150,6 @@ impl Stream for EventChunkerMultifile {
self.range.clone(),
self.event_chunker_conf.clone(),
path,
self.max_ts.clone(),
self.expand,
self.do_decompress,
);
@@ -161,7 +182,6 @@ impl Stream for EventChunkerMultifile {
self.range.clone(),
self.event_chunker_conf.clone(),
of.path,
self.max_ts.clone(),
self.expand,
self.do_decompress,
);
@@ -184,7 +204,7 @@ impl Stream for EventChunkerMultifile {
let item = LogItem::quick(
Level::INFO,
format!(
"EventBlobsComplete used {} datafiles beg {} end {} node_ix {}",
"EventChunkerMultifile used {} datafiles beg {} end {} node_ix {}",
self.files_count,
self.range.beg / SEC,
self.range.end / SEC,

58
disk/src/eventchunker.rs
View File

@@ -15,8 +15,6 @@ use parse::channelconfig::CompressionMethod;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;
@@ -35,7 +33,7 @@ pub struct EventChunker {
final_stats_sent: bool,
parsed_bytes: u64,
dbg_path: PathBuf,
max_ts: Arc<AtomicU64>,
max_ts: u64,
expand: bool,
do_decompress: bool,
decomp_dt_histo: HistoLog2,
@@ -43,6 +41,7 @@ pub struct EventChunker {
seen_before_range_count: usize,
seen_after_range_count: usize,
unordered_warn_count: usize,
repeated_ts_warn_count: usize,
}
impl Drop for EventChunker {
@@ -84,7 +83,6 @@ impl EventChunker {
range: NanoRange,
stats_conf: EventChunkerConf,
dbg_path: PathBuf,
max_ts: Arc<AtomicU64>,
expand: bool,
do_decompress: bool,
) -> Self {
@@ -106,7 +104,7 @@ impl EventChunker {
final_stats_sent: false,
parsed_bytes: 0,
dbg_path,
max_ts,
max_ts: 0,
expand,
do_decompress,
decomp_dt_histo: HistoLog2::new(8),
@@ -114,6 +112,7 @@ impl EventChunker {
seen_before_range_count: 0,
seen_after_range_count: 0,
unordered_warn_count: 0,
repeated_ts_warn_count: 0,
}
}
@@ -124,20 +123,10 @@ impl EventChunker {
range: NanoRange,
stats_conf: EventChunkerConf,
dbg_path: PathBuf,
max_ts: Arc<AtomicU64>,
expand: bool,
do_decompress: bool,
) -> Self {
let mut ret = Self::from_start(
inp,
channel_config,
range,
stats_conf,
dbg_path,
max_ts,
expand,
do_decompress,
);
let mut ret = Self::from_start(inp, channel_config, range, stats_conf, dbg_path, expand, do_decompress);
ret.state = DataFileState::Event;
ret.need_min = 4;
ret.inp.set_need_min(4);
@@ -205,24 +194,41 @@ impl EventChunker {
let _ttl = sl.read_i64::<BE>().unwrap();
let ts = sl.read_i64::<BE>().unwrap() as u64;
let pulse = sl.read_i64::<BE>().unwrap() as u64;
let max_ts = self.max_ts.load(Ordering::SeqCst);
if ts < max_ts {
if ts == self.max_ts {
if self.repeated_ts_warn_count < 20 {
let msg = format!(
"EventChunker repeated event ts ix {} ts {}.{} max_ts {}.{} config {:?} path {:?}",
self.repeated_ts_warn_count,
ts / SEC,
ts % SEC,
self.max_ts / SEC,
self.max_ts % SEC,
self.channel_config.shape,
self.dbg_path
);
warn!("{}", msg);
self.repeated_ts_warn_count += 1;
}
}
if ts < self.max_ts {
if self.unordered_warn_count < 20 {
let msg = format!(
"unordered event no {} ts: {}.{} max_ts {}.{} config {:?} path {:?}",
"EventChunker unordered event ix {} ts {}.{} max_ts {}.{} config {:?} path {:?}",
self.unordered_warn_count,
ts / SEC,
ts % SEC,
max_ts / SEC,
max_ts % SEC,
self.max_ts / SEC,
self.max_ts % SEC,
self.channel_config.shape,
self.dbg_path
);
warn!("{}", msg);
self.unordered_warn_count += 1;
let e = Error::with_public_msg_no_trace(msg);
return Err(e);
}
}
self.max_ts.store(ts, Ordering::SeqCst);
self.max_ts = ts;
if ts >= self.range.end {
self.seen_after_range_count += 1;
if !self.expand || self.seen_after_range_count >= 2 {
@@ -234,8 +240,8 @@ impl EventChunker {
if ts < self.range.beg {
self.seen_before_range_count += 1;
if self.seen_before_range_count > 1 {
let e = Error::with_msg(format!(
"seen before range: event ts: {}.{} range beg: {}.{} range end: {}.{} pulse {} config {:?} path {:?}",
let msg = format!(
"seen before range: event ts {}.{} range beg {}.{} range end {}.{} pulse {} config {:?} path {:?}",
ts / SEC,
ts % SEC,
self.range.beg / SEC,
@@ -245,7 +251,9 @@ impl EventChunker {
pulse,
self.channel_config.shape,
self.dbg_path
));
);
warn!("{}", msg);
let e = Error::with_public_msg(msg);
Err(e)?;
}
}

9
disk/src/merge.rs
View File

@@ -220,7 +220,7 @@ where
if lowest_ts < self.ts_last_emit {
self.errored = true;
let msg = format!("unordered event at lowest_ts {}", lowest_ts);
return Ready(Some(Err(Error::with_msg(msg))));
return Ready(Some(Err(Error::with_public_msg(msg))));
} else {
self.ts_last_emit = self.ts_last_emit.max(lowest_ts);
}
@@ -296,12 +296,11 @@ mod test {
use err::Error;
use futures_util::StreamExt;
use items::{RangeCompletableItem, StreamItem};
use netpod::log::*;
use netpod::test_data_base_path_databuffer;
use netpod::timeunits::{DAY, MS};
use netpod::{log::*, test_data_base_path_databuffer};
use netpod::{ByteOrder, ByteSize, Channel, ChannelConfig, FileIoBufferSize, NanoRange, Nanos, ScalarType, Shape};
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
fn scalar_file_path() -> PathBuf {
test_data_base_path_databuffer()
@@ -358,7 +357,6 @@ mod test {
let stats_conf = EventChunkerConf {
disk_stats_every: ByteSize::kb(1024),
};
let max_ts = Arc::new(AtomicU64::new(0));
let expand = false;
let do_decompress = false;
let dbg_path = PathBuf::from("/dbg/dummy");
@@ -372,7 +370,6 @@ mod test {
range.clone(),
stats_conf,
dbg_path,
max_ts,
expand,
do_decompress,
);