daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unused archiver file format support

Browse Source
This commit is contained in:
Dominik Werder
2022-07-26 12:06:00 +02:00
parent f195a941f1
commit 41652fe3f9
37 changed files with 16 additions and 16545 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
members = ["daqbuffer", "h5out", "archapp", "archapp_xc", "archapp_wrap", "items", "items_proc", "nodenet", "httpclient", "fsio", "dq"]
members = ["daqbuffer", "h5out", "items", "items_proc", "nodenet", "httpclient", "fsio", "dq"]
[profile.release]
opt-level = 1
archapp/Cargo.toml
-36
View File
@@ -1,36 +0,0 @@
[package]
name = "archapp"
version = "0.0.1-a.dev.4"
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2021"
[dependencies]
tokio = { version = "1.7.1", features = ["io-util", "net", "time", "sync", "fs", "parking_lot"] }
tracing = "0.1.26"
futures-core = "0.3.15"
futures-util = "0.3.15"
bytes = "1.0.1"
serde = "1.0.126"
serde_derive = "1.0.126"
serde_json = "1.0.64"
bincode = "1.3.3"
chrono = "0.4.19"
protobuf = "2.24.1"
async-channel = "1.6"
parking_lot = "0.11.2"
crc32fast = "1.2.1"
regex = "1.5.4"
tokio-postgres = { version = "0.7.4", features = ["runtime", "with-chrono-0_4", "with-serde_json-1"] }
archapp_xc = { path = "../archapp_xc" }
err = { path = "../err" }
taskrun = { path = "../taskrun" }
netpod = { path = "../netpod" }
dbconn = { path = "../dbconn" }
items = { path = "../items" }
items_proc = { path = "../items_proc" }
streams = { path = "../streams" }
commonio = { path = "../commonio" }
[features]
default = ["devread"]
devread = []
archapp/src/archeng.rs
-402
View File
@@ -1,402 +0,0 @@
pub mod backreadbuf;
pub mod blockrefstream;
pub mod blockstream;
pub mod bufminread;
pub mod configs;
pub mod datablock;
pub mod diskio;
pub mod indexfiles;
pub mod indextree;
pub mod pipe;
use self::indexfiles::{database_connect, list_index_files};
use self::indextree::channel_list;
use crate::timed::Timed;
use crate::wrap_task;
use async_channel::{Receiver, Sender};
use commonio::StatsChannel;
use err::{ErrStr, Error};
use futures_util::StreamExt;
use items::{StreamItem, WithLen};
use netpod::timeunits::SEC;
use netpod::{log::*, Database};
use netpod::{ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse};
use netpod::{ScalarType, Shape};
use serde::Serialize;
use serde_json::Value as JsVal;
use std::convert::TryInto;
const EPICS_EPOCH_OFFSET: u64 = 631152000 * SEC;
pub fn name_hash(s: &str, ht_len: u32) -> u32 {
let mut h = 0;
for ch in s.as_bytes() {
h = (128 * h + *ch as u32) % ht_len;
}
h
}
fn format_hex_block(buf: &[u8], max: usize) -> String {
use std::fmt::Write;
const COLS: usize = 16;
let buf = if buf.len() > max { &buf[0..max] } else { buf };
let mut i1 = 0;
let mut ret = String::new();
while i1 < buf.len() {
buf[i1..i1 + COLS].iter().for_each(|x| {
write!(&mut ret, " {:02x}", *x).unwrap();
});
ret.push('\n');
i1 += COLS;
}
ret
}
fn readu64(buf: &[u8], pos: usize) -> u64 {
u64::from_be_bytes(buf.as_ref()[pos..pos + 8].try_into().unwrap())
}
fn readu32(buf: &[u8], pos: usize) -> u32 {
u32::from_be_bytes(buf.as_ref()[pos..pos + 4].try_into().unwrap())
}
fn readu16(buf: &[u8], pos: usize) -> u16 {
u16::from_be_bytes(buf.as_ref()[pos..pos + 2].try_into().unwrap())
}
fn readf64(buf: &[u8], pos: usize) -> f64 {
f64::from_be_bytes(buf.as_ref()[pos..pos + 8].try_into().unwrap())
}
fn read_string(buf: &[u8]) -> Result<String, Error> {
let imax = buf
.iter()
.map(|k| *k)
.enumerate()
.take_while(|&(_, k)| k != 0)
.last()
.map(|(i, _)| i);
let ret = match imax {
Some(imax) => String::from_utf8(buf[..imax + 1].to_vec())?,
None => String::new(),
};
Ok(ret)
}
#[allow(dead_code)]
async fn datarange_stream_fill(_channel_name: &str, _tx: Sender<Datarange>) {
// Search the first relevant leaf node.
// Pipe all ranges from there, and continue with nodes.
// Issue: can not stop because I don't look into the files.
}
// TODO
// Should contain enough information to allow one to open and position a relevant datafile.
pub struct Datarange {}
pub fn datarange_stream(_channel_name: &str) -> Result<Receiver<Datarange>, Error> {
let (_tx, rx) = async_channel::bounded(4);
let task = async {};
taskrun::spawn(task);
Ok(rx)
}
#[derive(Debug, Serialize)]
pub struct ListChannelItem {
name: String,
index_path: String,
matches: bool,
}
pub fn list_all_channels(node: &ChannelArchiver) -> Receiver<Result<ListChannelItem, Error>> {
let node = node.clone();
let (tx, rx) = async_channel::bounded(4);
let tx2 = tx.clone();
let stats = {
let (tx, rx) = async_channel::bounded(16);
taskrun::spawn(async move {
let mut rx = rx;
while let Some(item) = rx.next().await {
match item {
Ok(StreamItem::Stats(item)) => {
debug!("stats: {:?}", item);
}
_ => {}
}
}
});
StatsChannel::new(tx.clone())
};
let task = async move {
let mut ixf = list_index_files(&node);
while let Some(f) = ixf.next().await {
let index_path = f?;
//info!("try to read for {:?}", index_path);
let channels = channel_list(index_path.clone(), &stats).await?;
//info!("list_all_channels emit {} channels", channels.len());
for ch in channels {
let mm = match ch.split("-").next() {
Some(k) => {
let dname = index_path.parent().unwrap().file_name().unwrap().to_str().unwrap();
if dname.starts_with(&format!("archive_{}", k)) {
true
} else {
false
}
}
None => false,
};
let item = ListChannelItem {
name: ch,
index_path: index_path.to_str().unwrap().into(),
matches: mm,
};
tx.send(Ok(item)).await.errstr()?;
//info!("{:?} parent {:?} channel {}", index_path, index_path.parent(), ch);
//break;
}
}
Ok::<_, Error>(())
};
wrap_task(task, tx2);
rx
}
struct ErrWrap(tokio_postgres::Error);
impl From<tokio_postgres::Error> for ErrWrap {
fn from(x: tokio_postgres::Error) -> Self {
Self(x)
}
}
impl From<ErrWrap> for Error {
fn from(_: ErrWrap) -> Self {
todo!()
}
}
pub async fn channel_config_from_db(
q: &ChannelConfigQuery,
_conf: &ChannelArchiver,
database: &Database,
) -> Result<ChannelConfigResponse, Error> {
let dbc = database_connect(database).await?;
let sql = "select config from channels where name = $1";
let rows = dbc.query(sql, &[&q.channel.name()]).await.errstr()?;
if let Some(row) = rows.first() {
let cfg: JsVal = row.try_get(0).errstr()?;
let val = cfg
.get("shape")
.ok_or_else(|| Error::with_msg_no_trace("shape not found on config"))?;
let shape = Shape::from_db_jsval(val)?;
let val = cfg
.get("scalarType")
.ok_or_else(|| Error::with_msg_no_trace("no scalarType in db"))?;
let s = if let JsVal::String(s) = val {
s
} else {
return Err(Error::with_msg_no_trace(format!(
"channel_config_from_db bad scalar type {:?}",
cfg
)));
};
let scalar_type = ScalarType::from_archeng_db_str(s)?;
let ret = ChannelConfigResponse {
channel: q.channel.clone(),
scalar_type,
// TODO.. only binary endpoint would care.
byte_order: None,
shape,
};
Ok(ret)
} else {
Err(Error::with_msg_no_trace(format!(
"can not find config for {}",
q.channel.name()
)))
}
}
pub async fn channel_config(
q: &ChannelConfigQuery,
_conf: &ChannelArchiver,
database: &Database,
) -> Result<ChannelConfigResponse, Error> {
let _timed = Timed::new("channel_config");
let mut type_info = None;
let ixpaths = indexfiles::index_file_path_list(q.channel.clone(), database.clone()).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let stream = blockrefstream::blockref_stream(q.channel.clone(), q.range.clone(), q.expand, ixpath.clone());
let stream = Box::pin(stream);
let stream = blockstream::BlockStream::new(stream, q.range.clone(), 1);
let mut stream = stream;
while let Some(item) = stream.next().await {
use blockstream::BlockItem::*;
match item {
Ok(k) => match k {
EventsItem(item) => {
if item.len() > 0 {
type_info = Some(item.type_info());
break;
}
}
JsVal(jsval) => {
debug!("jsval: {}", serde_json::to_string(&jsval)?);
}
},
Err(e) => {
error!("{}", e);
()
}
}
}
if type_info.is_none() {
let timed_normal = Timed::new("channel_config NORMAL");
warn!("channel_config expand mode returned none");
let stream = blockrefstream::blockref_stream(q.channel.clone(), q.range.clone(), q.expand, ixpath.clone());
let stream = Box::pin(stream);
let stream = blockstream::BlockStream::new(stream, q.range.clone(), 1);
let mut stream = stream;
while let Some(item) = stream.next().await {
use blockstream::BlockItem::*;
match item {
Ok(k) => match k {
EventsItem(item) => {
if item.len() > 0 {
type_info = Some(item.type_info());
break;
}
}
JsVal(jsval) => {
debug!("jsval: {}", serde_json::to_string(&jsval)?);
}
},
Err(e) => {
error!("{}", e);
()
}
}
}
drop(timed_normal);
}
if let Some(type_info) = type_info {
let ret = ChannelConfigResponse {
channel: q.channel.clone(),
scalar_type: type_info.0,
byte_order: None,
shape: type_info.1,
};
Ok(ret)
} else {
Err(Error::with_msg_no_trace("can not get channel type info"))
}
}
#[cfg(test)]
mod test {
use crate::archeng::datablock::{read_data_1, read_datafile_header};
use crate::archeng::indextree::{read_channel, read_datablockref, search_record};
use crate::archeng::{StatsChannel, EPICS_EPOCH_OFFSET};
use commonio::open_read;
use err::Error;
use items::{LogItem, Sitemty, StatsItem, StreamItem};
use netpod::timeunits::*;
use netpod::{log::*, RangeFilterStats};
use netpod::{FilePos, NanoRange, Nanos};
use serde::Serialize;
use std::path::PathBuf;
/*
Root node: most left record ts1 965081099942616289, most right record ts2 1002441959876114632
*/
const CHN_0_MASTER_INDEX: &str = "/data/daqbuffer-testdata/sls/gfa03/bl_arch/archive_X05DA_SH/index";
#[test]
fn search_record_data() -> Result<(), Error> {
let fut = async {
let stats = &StatsChannel::dummy();
let index_path: PathBuf = CHN_0_MASTER_INDEX.into();
let index_file = open_read(index_path.clone(), stats).await?;
let mut file2 = open_read(index_path.clone(), stats).await?;
let channel_name = "X05DA-FE-WI1:TC1";
const T0: u64 = 1002000000 * SEC + EPICS_EPOCH_OFFSET;
let beg = Nanos { ns: T0 };
let range = NanoRange {
beg: beg.ns,
end: beg.ns + 20 * SEC,
};
let res = read_channel(index_path.clone(), index_file, channel_name, stats).await?;
let cib = res.unwrap();
let (res, _stats) = search_record(&mut file2, cib.rtree_m, cib.rtree_start_pos, beg, stats).await?;
assert_eq!(res.is_some(), true);
let res = res.unwrap();
assert_eq!(res.node.is_leaf, true);
assert_eq!(res.node.pos.pos, 1861178);
assert_eq!(res.rix, 41);
let rec = &res.node.records[res.rix];
assert_eq!(rec.ts1.ns, 1001993759871202919 + EPICS_EPOCH_OFFSET);
assert_eq!(rec.ts2.ns, 1002009299596362122 + EPICS_EPOCH_OFFSET);
assert_eq!(rec.child_or_id, 2501903);
let pos = FilePos { pos: rec.child_or_id };
let datablock = read_datablockref(&mut file2, pos, cib.hver(), stats).await?;
assert_eq!(datablock.data_header_pos().0, 9311367);
assert_eq!(datablock.file_name(), "20211001/20211001");
let data_path = index_path.parent().unwrap().join(datablock.file_name());
let mut data_file = open_read(data_path, stats).await?;
let datafile_header = read_datafile_header(&mut data_file, datablock.data_header_pos(), stats).await?;
let events = read_data_1(&mut data_file, &datafile_header, range.clone(), false, stats).await?;
debug!("read events: {:?}", events);
// TODO assert more
Ok(())
};
Ok(taskrun::run(fut).unwrap())
}
#[test]
fn test_bincode_rep_stats() {
fn make_stats<T>() -> Vec<u8>
where
T: Serialize,
{
let stats = RangeFilterStats {
events_pre: 626262,
events_post: 929292,
events_unordered: 131313,
};
let item = StreamItem::Stats(StatsItem::RangeFilterStats(stats));
let item: Sitemty<T> = Ok(item);
bincode::serialize(&item).unwrap()
}
let v1 = make_stats::<u8>();
let v2 = make_stats::<f32>();
let v3 = make_stats::<Vec<u32>>();
let v4 = make_stats::<Vec<f64>>();
assert_eq!(v1, v2);
assert_eq!(v1, v3);
assert_eq!(v1, v4);
}
#[test]
fn test_bincode_rep_log() {
fn make_log<T>() -> Vec<u8>
where
T: Serialize,
{
let item = StreamItem::Log(LogItem::quick(
Level::DEBUG,
format!("Some easy log message for testing purpose here."),
));
let item: Sitemty<T> = Ok(item);
bincode::serialize(&item).unwrap()
}
let v1 = make_log::<u8>();
let v2 = make_log::<f32>();
let v3 = make_log::<Vec<u32>>();
let v4 = make_log::<Vec<f64>>();
assert_eq!(v1, v2);
assert_eq!(v1, v3);
assert_eq!(v1, v4);
}
}
archapp/src/archeng/backreadbuf.rs
-139
View File
@@ -1,139 +0,0 @@
use commonio::{read, seek, StatsChannel};
use err::Error;
use netpod::log::*;
use std::borrow::BorrowMut;
use std::fmt;
use std::io::SeekFrom;
use tokio::fs::File;
pub struct BackReadBuf<F> {
file: F,
buf: Vec<u8>,
abs: u64,
wp: usize,
rp: usize,
stats: StatsChannel,
seek_request: u64,
seek_done: u64,
read_done: u64,
bytes_read: u64,
}
impl<F> BackReadBuf<F>
where
F: BorrowMut<File>,
{
pub async fn new(file: F, pos: u64, stats: StatsChannel) -> Result<Self, Error> {
let mut ret = Self {
file,
buf: vec![0; 1024 * 8],
abs: pos,
wp: 0,
rp: 0,
stats,
seek_request: 0,
seek_done: 0,
read_done: 0,
bytes_read: 0,
};
ret.seek(pos).await?;
Ok(ret)
}
pub fn into_file(self) -> F {
//self.file
err::todoval()
}
pub fn len(&self) -> usize {
self.wp - self.rp
}
pub fn adv(&mut self, n: usize) {
self.rp += n;
}
pub fn data(&self) -> &[u8] {
&self.buf[self.rp..self.wp]
}
async fn fill(&mut self) -> Result<usize, Error> {
if self.rp != 0 && self.rp == self.wp {
self.wp = 0;
self.rp = 0;
} else {
unsafe {
std::ptr::copy::<u8>(&self.buf[self.rp], &mut self.buf[0], self.len());
self.wp -= self.rp;
self.rp = 0;
}
}
let n = read(&mut self.file.borrow_mut(), &mut self.buf[self.wp..], &self.stats).await?;
//debug!("I/O fill n {}", n);
self.wp += n;
self.read_done += 1;
self.bytes_read += n as u64;
Ok(n)
}
pub async fn fill_min(&mut self, min: usize) -> Result<usize, Error> {
let len = self.len();
while self.len() < min {
let n = self.fill().await?;
if n == 0 {
return Err(Error::with_msg(format!("fill_min can not read min {}", min)));
}
}
Ok(self.len() - len)
}
pub async fn seek(&mut self, pos: u64) -> Result<u64, Error> {
if pos >= self.abs && pos < self.abs + self.buf.len() as u64 - 64 {
self.rp = (pos - self.abs) as usize;
self.seek_request += 1;
Ok(pos)
} else {
//debug!("I/O seek dp {}", dp);
let s0 = pos.min(1024 * 2 - 256);
self.abs = pos - s0;
self.rp = 0;
self.wp = 0;
let ret = seek(self.file.borrow_mut(), SeekFrom::Start(self.abs), &self.stats)
.await
.map_err(|e| Error::from(e))?;
self.fill_min(s0 as usize).await?;
self.rp = s0 as usize;
self.seek_request += 1;
self.seek_done += 1;
Ok(ret)
}
}
pub fn rp_abs(&self) -> u64 {
self.abs as u64 + self.rp as u64
}
pub fn bytes_read(&self) -> u64 {
self.bytes_read
}
}
impl<F> fmt::Debug for BackReadBuf<F> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("BackReadBuf")
.field("abs", &self.abs)
.field("wp", &self.wp)
.field("rp", &self.rp)
.field("seek_request", &self.seek_request)
.field("seek_done", &self.seek_done)
.field("read_done", &self.read_done)
.field("bytes_read", &self.bytes_read)
.finish()
}
}
impl<F> Drop for BackReadBuf<F> {
fn drop(&mut self) {
trace!("Drop {:?}", self);
}
}
archapp/src/archeng/blockrefstream.rs
-231
View File
@@ -1,231 +0,0 @@
use crate::archeng::backreadbuf::BackReadBuf;
use crate::archeng::indexfiles::{unfold_stream, UnfoldExec};
use crate::archeng::indextree::{
read_datablockref2, Dataref, HeaderVersion, IndexFileBasics, RecordIter, RecordTarget,
};
use commonio::{open_read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
#[allow(unused)]
use netpod::log::*;
use netpod::{Channel, NanoRange};
#[allow(unused)]
use serde::Serialize;
use serde_json::Value as JsVal;
use std::collections::VecDeque;
use std::path::PathBuf;
use std::pin::Pin;
use tokio::fs::File;
#[derive(Debug)]
pub struct Blockref {
pub dref: Dataref,
pub dpath: PathBuf,
}
#[derive(Debug)]
pub enum BlockrefItem {
Blockref(Blockref, JsVal),
JsVal(JsVal),
}
enum Steps {
Start,
SelectIndexFile,
SetupNextPath,
ReadBlocks(RecordIter, Box<dyn HeaderVersion>, PathBuf),
Done,
}
struct BlockrefStream {
channel: Channel,
range: NanoRange,
expand: bool,
steps: Steps,
paths: VecDeque<PathBuf>,
file1: Option<BackReadBuf<File>>,
last_dp: u64,
last_dp2: u64,
data_bytes_read: u64,
same_dfh_count: u64,
}
impl BlockrefStream {
fn new(channel: Channel, range: NanoRange, expand: bool, path: PathBuf) -> Self {
debug!("new BlockrefStream {:?} {:?}", range, path);
Self {
channel,
range,
expand,
steps: Steps::Start,
paths: VecDeque::from([path]),
file1: None,
last_dp: 0,
last_dp2: 0,
data_bytes_read: 0,
same_dfh_count: 0,
}
}
async fn exec(mut self) -> Result<Option<(BlockrefItem, Self)>, Error> {
use Steps::*;
match self.steps {
Start => {
self.steps = SelectIndexFile;
Ok(Some((
BlockrefItem::JsVal(JsVal::String(format!("{} START", module_path!()))),
self,
)))
}
SelectIndexFile => {
if self.paths.len() == 0 {
self.steps = Done;
Ok(Some((
BlockrefItem::JsVal(JsVal::String(format!("NOPATHANYMORE"))),
self,
)))
} else {
self.steps = SetupNextPath;
Ok(Some((BlockrefItem::JsVal(JsVal::String(format!("DBQUERY"))), self)))
}
}
SetupNextPath => {
let stats = &StatsChannel::dummy();
// For simplicity, simply read all storage classes linearly.
if let Some(path) = self.paths.pop_front() {
debug!("SetupNextPath {:?}", path);
// TODO
let mut file = open_read(path.clone(), stats).await.map_err(|e| {
error!("can not open {:?}", path);
e
})?;
let basics = IndexFileBasics::from_file(&path, &mut file, stats).await?;
let mut tree = basics
.rtree_for_channel(self.channel.name(), stats)
.await?
.ok_or_else(|| Error::with_msg_no_trace("channel not in index files"))?;
if let Some(iter) = tree.iter_range(self.range.clone(), self.expand, stats).await? {
debug!("SetupNextPath {:?}", path);
self.steps = ReadBlocks(iter, basics.hver().duplicate(), path.clone().into());
self.file1 = Some(BackReadBuf::new(file, 0, stats.clone()).await?);
} else {
self.steps = SetupNextPath;
};
Ok(Some((BlockrefItem::JsVal(JsVal::String(format!("NEXTPATH"))), self)))
} else {
self.steps = Done;
Ok(Some((
BlockrefItem::JsVal(JsVal::String(format!("PATHQUEUEEMPTY"))),
self,
)))
}
}
ReadBlocks(ref mut iter, ref hver, ref indexpath) => {
let item = if let Some(rec) = iter.next().await? {
// TODO the iterator should actually return Dataref. We never expect child nodes here.
if let RecordTarget::Dataref(dp) = rec.target {
let f1 = self.file1.as_mut().unwrap();
let dref = read_datablockref2(f1, dp.clone(), hver.as_ref()).await?;
let dpath = indexpath.parent().unwrap().join(dref.file_name());
let jsval = serde_json::to_value((
dp.0,
dp.0 as i64 - self.last_dp as i64,
dref.file_name(),
dref.data_header_pos.0,
dref.data_header_pos.0 as i64 - self.last_dp2 as i64,
dref.next().0,
))?;
self.last_dp = dp.0;
self.last_dp2 = dref.data_header_pos.0;
if rec.end.ns > self.range.end {
debug!("Have block end beyond range, stop");
self.steps = Done;
}
let bref = Blockref { dref, dpath };
trace!("emit {:?} Record range: {:?} TO {:?}", bref, rec.beg, rec.end);
BlockrefItem::Blockref(bref, jsval)
} else {
error!("not a Dataref target");
self.steps = Done;
BlockrefItem::JsVal(JsVal::String(format!("not a Dataref target")))
}
} else {
debug!(
"data_bytes_read: {} same_dfh_count: {}",
self.data_bytes_read, self.same_dfh_count
);
self.steps = SetupNextPath;
BlockrefItem::JsVal(JsVal::String(format!("NOMORE")))
};
Ok(Some((item, self)))
}
Done => Ok(None),
}
}
}
impl UnfoldExec for BlockrefStream {
type Output = BlockrefItem;
fn exec(self) -> Pin<Box<dyn Future<Output = Result<Option<(Self::Output, Self)>, Error>> + Send>>
where
Self: Sized,
{
Box::pin(self.exec())
}
}
pub fn blockref_stream(
channel: Channel,
range: NanoRange,
expand: bool,
ixpath: PathBuf,
) -> impl Stream<Item = Result<BlockrefItem, Error>> {
unfold_stream(BlockrefStream::new(channel, range, expand, ixpath))
}
#[cfg(test)]
mod test {
use super::*;
use crate::archeng::indexfiles::index_file_path_list;
use futures_util::StreamExt;
use netpod::timeunits::SEC;
use netpod::Database;
#[test]
fn find_ref_1() -> Result<(), err::Error> {
let fut = async move {
let channel = Channel {
backend: "sls-archive".into(),
name: "X05DA-FE-WI1:TC1".into(),
series: None,
};
use chrono::{DateTime, Utc};
let dtbeg: DateTime<Utc> = "2021-10-01T00:00:00Z".parse()?;
let dtend: DateTime<Utc> = "2021-10-10T00:00:00Z".parse()?;
fn tons(dt: &DateTime<Utc>) -> u64 {
dt.timestamp() as u64 * SEC + dt.timestamp_subsec_nanos() as u64
}
let range = NanoRange {
beg: tons(&dtbeg),
end: tons(&dtend),
};
let dbconf = Database {
host: "localhost".into(),
port: 5432,
name: "testingdaq".into(),
user: "testingdaq".into(),
pass: "testingdaq".into(),
};
let ixpaths = index_file_path_list(channel.clone(), dbconf).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let mut refs = Box::pin(blockref_stream(channel, range, false, ixpath));
while let Some(item) = refs.next().await {
info!("Got ref {:?}", item);
}
Ok(())
};
taskrun::run(fut)
}
}
archapp/src/archeng/blockstream.rs
-775
View File
@@ -1,775 +0,0 @@
use crate::archeng::blockrefstream::BlockrefItem;
use crate::archeng::datablock::{read_data2, read_datafile_header2};
use crate::archeng::indextree::DataheaderPos;
use commonio::ringbuf::RingBuf;
use commonio::{open_read, StatsChannel};
use err::Error;
use futures_core::{Future, Stream};
use futures_util::stream::FuturesOrdered;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::{WithLen, WithTimestamps};
use netpod::{log::*, NanoRange, Nanos};
use serde::Serialize;
use serde_json::Value as JsVal;
use std::collections::{BTreeMap, VecDeque};
use std::fmt;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tokio::fs::File;
#[derive(Debug, Serialize)]
pub struct StatsAcc {
items: u64,
events: u64,
bytes: u64,
#[serde(skip)]
beg: Instant,
}
impl StatsAcc {
pub fn new() -> Self {
Self {
items: 0,
events: 0,
bytes: 0,
beg: Instant::now(),
}
}
fn add(&mut self, events: u64, bytes: u64) {
self.items += 1;
self.events += events;
self.bytes += bytes;
}
fn older(&self, dur: Duration) -> bool {
Instant::now().duration_since(self.beg) >= dur
}
}
struct Reader {
fname: String,
rb: RingBuf<File>,
}
impl Reader {}
struct FutAItem {
fname: String,
path: PathBuf,
dpos: DataheaderPos,
dfnotfound: bool,
reader: Option<Reader>,
bytes_read: u64,
events_read: u64,
events: Option<EventsItem>,
}
#[allow(unused)]
pub struct FutA {
fname: String,
pos: DataheaderPos,
reader: Option<Reader>,
}
impl Future for FutA {
type Output = Result<JsVal, Error>;
#[allow(unused)]
fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
use Poll::*;
err::todoval()
}
}
#[derive(Debug)]
pub enum BlockItem {
EventsItem(EventsItem),
JsVal(JsVal),
}
pub struct BlockStream<S> {
inp: S,
inp_done: bool,
range: NanoRange,
dfnotfound: BTreeMap<PathBuf, bool>,
block_reads: FuturesOrdered<Pin<Box<dyn Future<Output = Result<FutAItem, Error>> + Send>>>,
max_reads: usize,
readers: VecDeque<Reader>,
last_dfname: String,
last_dfhpos: DataheaderPos,
ts_max: u64,
data_done: bool,
raco: bool,
done: bool,
complete: bool,
acc: StatsAcc,
good_reader: u64,
discard_reader: u64,
not_found_hit: u64,
same_block: u64,
}
impl<S> BlockStream<S> {
pub fn new(inp: S, range: NanoRange, max_reads: usize) -> Self
where
S: Stream<Item = Result<BlockrefItem, Error>> + Unpin,
{
debug!("new BlockStream max_reads {} {:?}", max_reads, range);
Self {
inp,
inp_done: false,
range,
dfnotfound: BTreeMap::new(),
block_reads: FuturesOrdered::new(),
max_reads: max_reads.max(1),
readers: VecDeque::new(),
last_dfname: String::new(),
last_dfhpos: DataheaderPos(u64::MAX),
ts_max: 0,
data_done: false,
raco: false,
done: false,
complete: false,
acc: StatsAcc::new(),
good_reader: 0,
discard_reader: 0,
not_found_hit: 0,
same_block: 0,
}
}
}
enum Int<T> {
NoWork,
Pending,
Empty,
Item(T),
Done,
}
impl<S> Stream for BlockStream<S>
where
S: Stream<Item = Result<BlockrefItem, Error>> + Unpin,
{
type Item = Result<BlockItem, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.complete {
panic!("poll_next on complete")
} else if self.done {
self.complete = true;
Ready(None)
} else if self.data_done {
self.done = true;
if self.raco {
// currently handled downstream
continue;
} else {
continue;
}
} else {
let item1 = if self.inp_done {
Int::Done
} else if self.block_reads.len() >= self.max_reads {
Int::NoWork
} else {
match self.inp.poll_next_unpin(cx) {
Ready(item) => match item {
Some(item) => match item {
Ok(item) => match item {
BlockrefItem::Blockref(bref, _jsval) => {
if let Some(_) = self.dfnotfound.get(&bref.dpath) {
self.not_found_hit += 1;
} else {
if bref.dref.file_name() == self.last_dfname
&& bref.dref.data_header_pos() == self.last_dfhpos
{
self.same_block += 1;
} else {
let reader = if let Some(reader) = self.readers.pop_front() {
if reader.fname == bref.dref.file_name() {
self.good_reader += 1;
Some(reader)
} else {
self.discard_reader += 1;
None
}
} else {
None
};
let fname = bref.dref.file_name().to_string();
let dpath = bref.dpath;
let pos = bref.dref.data_header_pos();
let fut = {
let fname = fname.clone();
let pos = pos.clone();
let range = self.range.clone();
async move {
let reader = if let Some(reader) = reader {
Some(reader)
} else {
let stats = StatsChannel::dummy();
trace!("open new reader file {:?}", dpath);
match open_read(dpath.clone(), &stats).await {
Ok(file) => {
//
let reader = Reader {
fname: fname.clone(),
rb: RingBuf::new(file, pos.0, stats).await?,
};
Some(reader)
}
Err(_) => None,
}
};
if let Some(mut reader) = reader {
let rp1 = reader.rb.bytes_read();
let dfheader =
read_datafile_header2(&mut reader.rb, pos.clone())
.await?;
// TODO handle expand
let expand = false;
let data =
read_data2(&mut reader.rb, &dfheader, range, expand)
.await
.map_err(|e| {
Error::with_msg_no_trace(format!(
"dpath {:?} error {}",
dpath, e
))
})?;
let rp2 = reader.rb.bytes_read();
let bytes_read = rp2 - rp1;
let ret = FutAItem {
fname,
path: dpath,
dpos: pos,
dfnotfound: false,
reader: Some(reader),
bytes_read,
events_read: data.len() as u64,
events: Some(data),
};
Ok(ret)
} else {
let ret = FutAItem {
fname,
path: dpath,
dpos: pos,
dfnotfound: true,
reader: None,
bytes_read: 0,
events_read: 0,
events: None,
};
Ok(ret)
}
}
};
self.block_reads.push(Box::pin(fut));
self.last_dfname = fname;
self.last_dfhpos = pos;
};
}
Int::Empty
}
BlockrefItem::JsVal(jsval) => Int::Item(Ok(BlockItem::JsVal(jsval))),
},
Err(e) => {
self.done = true;
Int::Item(Err(e))
}
},
None => {
self.inp_done = true;
Int::Done
}
},
Pending => Int::Pending,
}
};
let item2 = if let Int::Item(_) = item1 {
Int::NoWork
} else {
if self.block_reads.len() == 0 {
Int::NoWork
} else {
match self.block_reads.poll_next_unpin(cx) {
Ready(Some(Ok(item))) => {
let mut item = item;
item.events = if let Some(ev) = item.events {
if ev.len() > 0 {
if ev.ts(ev.len() - 1) > self.range.end {
debug!(". . . . ===== DATA DONE ----------------------");
self.raco = true;
self.data_done = true;
}
}
if ev.len() == 1 {
trace!("From {} {:?} {}", item.fname, item.path, item.dpos.0);
trace!("See 1 event {:?}", Nanos::from_ns(ev.ts(0)));
} else if ev.len() > 1 {
trace!("From {} {:?} {}", item.fname, item.path, item.dpos.0);
trace!(
"See {} events {:?} to {:?}",
ev.len(),
Nanos::from_ns(ev.ts(0)),
Nanos::from_ns(ev.ts(ev.len() - 1))
);
}
let mut contains_unordered = false;
for i in 0..ev.len() {
// TODO factor for performance.
let ts = ev.ts(i);
if ts < self.ts_max {
contains_unordered = true;
if true {
let msg = format!(
"unordered event in item at {} ts {:?} ts_max {:?}",
i,
Nanos::from_ns(ts),
Nanos::from_ns(self.ts_max)
);
error!("{}", msg);
self.done = true;
return Ready(Some(Err(Error::with_msg_no_trace(msg))));
}
}
self.ts_max = ts;
}
if contains_unordered {
Some(ev)
} else {
Some(ev)
}
} else {
None
};
let item = item;
if item.dfnotfound {
self.dfnotfound.insert(item.path.clone(), true);
}
if let Some(reader) = item.reader {
self.readers.push_back(reader);
}
self.acc.add(item.events_read, item.bytes_read);
if false {
let item = JsVal::String(format!(
"bytes read {} {} events {}",
item.bytes_read,
item.events.is_some(),
item.events_read
));
let _ = item;
}
if false {
// TODO emit proper variant for optional performance measurement.
if self.acc.older(Duration::from_millis(1000)) {
let ret = std::mem::replace(&mut self.acc, StatsAcc::new());
match serde_json::to_value((ret, self.block_reads.len(), self.readers.len())) {
Ok(item) => Int::Item(Ok::<_, Error>(item)),
Err(e) => {
self.done = true;
return Ready(Some(Err(e.into())));
}
}
} else {
//Int::Item(Ok(item))
Int::Empty
};
err::todoval()
} else {
if let Some(events) = item.events {
Int::Item(Ok(BlockItem::EventsItem(events)))
} else {
Int::Empty
}
}
}
Ready(Some(Err(e))) => {
self.done = true;
error!("{}", e);
Int::Item(Err(e))
}
Ready(None) => {
panic!();
}
Pending => Int::Pending,
}
}
};
match (item1, item2) {
(Int::Item(_), Int::Item(_)) => panic!(),
(Int::NoWork, Int::NoWork) => panic!(),
(_, Int::Done) => panic!(),
(Int::Item(item), _) => Ready(Some(item)),
(_, Int::Item(item)) => Ready(Some(item)),
(Int::Pending | Int::NoWork, Int::Pending) => Pending,
(Int::Pending, Int::NoWork) => Pending,
(Int::Done, Int::Pending) => Pending,
(Int::Pending | Int::Done | Int::Empty | Int::NoWork, Int::Empty) => continue,
(Int::Empty, Int::Pending | Int::NoWork) => continue,
(Int::Done, Int::NoWork) => {
self.done = true;
Ready(None)
}
}
};
}
}
}
impl<S> fmt::Debug for BlockStream<S> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("BlockStream")
.field("inp_done", &self.inp_done)
.field("range", &self.range)
.field("max_reads", &self.max_reads)
.field("ts_max", &self.ts_max)
.field("done", &self.done)
.field("complete", &self.complete)
.field("acc", &self.acc)
.field("good_reader", &self.good_reader)
.field("discard_reader", &self.discard_reader)
.field("not_found_hit", &self.not_found_hit)
.field("same_block", &self.same_block)
.finish()
}
}
impl<S> Drop for BlockStream<S> {
fn drop(&mut self) {
trace!("Drop {:?}", self);
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::archeng::blockrefstream::blockref_stream;
use crate::archeng::indexfiles::index_file_path_list;
use chrono::{DateTime, Utc};
use futures_util::StreamExt;
use items::{LogItem, RangeCompletableItem, StreamItem};
use netpod::{timeunits::SEC, Channel, Database};
use streams::rangefilter::RangeFilter;
struct EventCount {
pre: usize,
inside: usize,
post: usize,
raco: usize,
tss: Vec<u64>,
}
impl EventCount {
fn new() -> Self {
Self {
pre: 0,
inside: 0,
post: 0,
raco: 0,
tss: vec![],
}
}
}
async fn count_events(range: NanoRange, expand: bool, collect_ts: bool) -> Result<EventCount, Error> {
let channel = Channel {
backend: "sls-archive".into(),
//name: "X05DA-FE-WI1:TC1".into(),
name: "ARIDI-PCT:CURRENT".into(),
series: None,
};
let dbconf = Database {
host: "localhost".into(),
port: 5432,
name: "testingdaq".into(),
user: "testingdaq".into(),
pass: "testingdaq".into(),
};
let ixpaths = index_file_path_list(channel.clone(), dbconf).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let refs = Box::pin(blockref_stream(channel, range.clone(), expand, ixpath));
let blocks = BlockStream::new(refs, range.clone(), 1);
let events = blocks.map(|item| match item {
Ok(k) => match k {
BlockItem::EventsItem(k) => Ok(StreamItem::DataItem(RangeCompletableItem::Data(k))),
BlockItem::JsVal(k) => Ok(StreamItem::Log(LogItem::quick(Level::TRACE, format!("{:?}", k)))),
},
Err(e) => Err(e),
});
let mut filtered = RangeFilter::new(events, range.clone(), expand);
let mut ret = EventCount::new();
while let Some(item) = filtered.next().await {
//info!("Got block {:?}", item);
match item {
Ok(item) => match item {
StreamItem::DataItem(item) => match item {
RangeCompletableItem::RangeComplete => {
ret.raco += 1;
}
RangeCompletableItem::Data(item) => {
let n = item.len();
for i in 0..n {
let ts = item.ts(i);
//info!("See event {}", ts);
if ts < range.beg {
ret.pre += 1;
} else if ts < range.end {
ret.inside += 1;
} else {
ret.post += 1;
}
if collect_ts {
ret.tss.push(ts);
}
}
}
},
StreamItem::Log(_) => {}
StreamItem::Stats(_) => {}
},
Err(e) => {
return Err(e);
}
}
}
Ok(ret)
}
fn tons(dt: &DateTime<Utc>) -> u64 {
dt.timestamp() as u64 * SEC + dt.timestamp_subsec_nanos() as u64
}
/*
See event 1636498894380250805
See event 1636499776028981476
See event 1636413555754299959
See event 1636413555908344145
See event 1636498896546533901
See event 1636498896546540966
See event 1636499855546054375
See event 1636499914581647548
See event 1636537592102377806
See event 1636545517217768432
See event 1636560318439777562
See event 1636585292173222036
See event 1636585292173229436
*/
#[test]
fn read_blocks_one_event_basic() -> Result<(), Error> {
//let ta = "2021-11-09T10:00:00Z";
//let tb = "2021-11-09T11:00:00Z";
let _ev1 = "2021-10-03T09:57:59.939651334Z";
let _ev2 = "2021-10-03T09:58:59.940910313Z";
let _ev3 = "2021-10-03T09:59:59.940112431Z";
// This is from bl SH index:
//let ev1ts = 1633255079939651334;
//let ev2ts = 1633255139940910313;
// [ev1..ev2]
//let beg = tons(&ta.parse()?);
//let end = tons(&tb.parse()?);
//let ev1ts = 1636498896546533901;
//let ev2ts = 1636498896546540966;
let beg = 1636455492985809049;
let end = 1636455493306756248;
let range = NanoRange { beg, end };
let res = taskrun::run(count_events(range, false, true))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 1);
assert_eq!(res.post, 0);
assert_eq!(res.tss[0], beg);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_one_event_expand() -> Result<(), Error> {
// This is from bl SH index:
//let ev1ts = 1633255079939651334;
//let ev2ts = 1633255139940910313;
let beg = 1636455492985809049;
let end = 1636455493306756248;
let range = NanoRange { beg, end };
let res = taskrun::run(count_events(range, true, true))?;
assert_eq!(res.pre, 1);
assert_eq!(res.inside, 1);
assert_eq!(res.post, 1);
assert_eq!(res.tss[1], beg);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_two_events_basic() -> Result<(), Error> {
let beg = 1636455492985809049;
let end = 1636455493306756248;
let range = NanoRange { beg: beg, end: end + 1 };
let res = taskrun::run(count_events(range, false, true))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 2);
assert_eq!(res.post, 0);
assert_eq!(res.tss[0], beg);
assert_eq!(res.tss[1], end);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_two_events_expand() -> Result<(), Error> {
let beg = 1636455492985809049;
let end = 1636455493306756248;
let range = NanoRange { beg: beg, end: end + 1 };
let res = taskrun::run(count_events(range, true, true))?;
assert_eq!(res.pre, 1);
assert_eq!(res.inside, 2);
assert_eq!(res.post, 1);
assert_eq!(res.tss[1], beg);
assert_eq!(res.tss[2], end);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_1_basic() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2021-11-09T11:00:00Z";
let dtbeg: DateTime<Utc> = dtbeg.parse()?;
let dtend: DateTime<Utc> = dtend.parse()?;
let range = NanoRange {
beg: tons(&dtbeg),
end: tons(&dtend),
};
let res = taskrun::run(count_events(range, false, false))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 726);
assert_eq!(res.post, 0);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_1_expand() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2021-11-09T11:00:00Z";
let dtbeg: DateTime<Utc> = dtbeg.parse()?;
let dtend: DateTime<Utc> = dtend.parse()?;
let range = NanoRange {
beg: tons(&dtbeg),
end: tons(&dtend),
};
let res = taskrun::run(count_events(range, true, false))?;
assert_eq!(res.pre, 1);
assert_eq!(res.inside, 726);
assert_eq!(res.post, 1);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_3_basic() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2021-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, false, false))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 2089);
assert_eq!(res.post, 0);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_3_expand() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2021-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, true, false))?;
assert_eq!(res.pre, 1);
assert_eq!(res.inside, 2089);
assert_eq!(res.post, 1);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_4_basic() -> Result<(), Error> {
let dtbeg = "2020-11-09T10:00:00Z";
let dtend = "2021-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, false, false))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 9518);
assert_eq!(res.post, 0);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_many_4_expand() -> Result<(), Error> {
let dtbeg = "2020-11-09T10:00:00Z";
let dtend = "2021-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, true, false))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 9518);
assert_eq!(res.post, 1);
assert_eq!(res.raco, 1);
Ok(())
}
#[test]
fn read_blocks_late_basic() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2022-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, false, false))?;
assert_eq!(res.pre, 0);
assert_eq!(res.inside, 12689);
assert_eq!(res.post, 0);
assert_eq!(res.raco, 0);
Ok(())
}
#[test]
fn read_blocks_late_expand() -> Result<(), Error> {
let dtbeg = "2021-11-09T10:00:00Z";
let dtend = "2022-11-09T13:00:00Z";
let range = NanoRange {
beg: tons(&dtbeg.parse()?),
end: tons(&dtend.parse()?),
};
let res = taskrun::run(count_events(range, true, false))?;
assert_eq!(res.pre, 1);
assert_eq!(res.inside, 12689);
assert_eq!(res.post, 0);
assert_eq!(res.raco, 0);
Ok(())
}
}
archapp/src/archeng/bufminread.rs
-1
View File
@@ -1 +0,0 @@
archapp/src/archeng/configs.rs
-272
View File
@@ -1,272 +0,0 @@
use crate::archeng::indexfiles::database_connect;
use err::{ErrStr, Error};
use futures_core::{Future, Stream};
use futures_util::{FutureExt, StreamExt};
use netpod::{log::*, NodeConfigCached};
use netpod::{Channel, ChannelArchiver, ChannelConfigQuery, ChannelConfigResponse, Database, NanoRange};
use serde::Serialize;
use serde_json::Value as JsVal;
use std::collections::VecDeque;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::{Duration, SystemTime};
use tokio_postgres::{Client, Row};
pub struct ChannelNameStream {
db_config: Database,
max_name: String,
db_done: bool,
batch: VecDeque<String>,
connect_fut: Option<Pin<Box<dyn Future<Output = Result<Client, Error>> + Send>>>,
select_fut: Option<Pin<Box<dyn Future<Output = Result<Vec<Row>, Error>> + Send>>>,
done: bool,
complete: bool,
}
impl ChannelNameStream {
pub fn new(db_config: Database) -> Self {
Self {
db_config,
max_name: String::new(),
db_done: false,
batch: VecDeque::new(),
connect_fut: None,
select_fut: None,
done: false,
complete: false,
}
}
}
impl Stream for ChannelNameStream {
type Item = Result<String, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.complete {
panic!("poll on complete")
} else if self.done {
self.complete = true;
Ready(None)
} else if let Some(item) = self.batch.pop_front() {
Ready(Some(Ok(item)))
} else if let Some(fut) = &mut self.select_fut {
match fut.poll_unpin(cx) {
Ready(Ok(rows)) => {
self.select_fut = None;
if rows.len() == 0 {
self.db_done = true;
}
if let Some(last) = rows.last().as_ref() {
self.max_name = last.get(1);
}
for row in rows {
self.batch.push_back(row.get(1));
}
continue;
}
Ready(Err(e)) => {
self.select_fut = None;
self.done = true;
Ready(Some(Err(e)))
}
Pending => Pending,
}
} else if let Some(fut) = &mut self.connect_fut {
match fut.poll_unpin(cx) {
Ready(Ok(dbc)) => {
self.connect_fut = None;
let max_name = self.max_name.clone();
info!("select channels max_name {}", max_name);
let fut = async move {
let rows = dbc
.query(
"select rowid, name from channels where config = '{}'::jsonb and name > $1 order by name limit 64",
&[&max_name],
)
.await.errstr()?;
Ok::<_, Error>(rows)
};
self.select_fut = Some(Box::pin(fut));
continue;
}
Ready(Err(e)) => {
self.connect_fut = None;
self.done = true;
Ready(Some(Err(e)))
}
Pending => Pending,
}
} else {
if self.db_done {
self.done = true;
info!("db_done");
continue;
} else {
let db = self.db_config.clone();
let fut = async move { database_connect(&db).await };
self.connect_fut = Some(Box::pin(fut));
continue;
}
};
}
}
}
enum Res {
TimedOut(String),
Response(ChannelConfigResponse),
}
#[derive(Debug, Serialize)]
pub enum ConfigItem {
Config(ChannelConfigResponse),
JsVal(JsVal),
}
pub struct ConfigStream {
node: NodeConfigCached,
conf: ChannelArchiver,
inp: ChannelNameStream,
inp_done: bool,
get_fut: Option<Pin<Box<dyn Future<Output = Result<Res, Error>> + Send>>>,
update_fut: Option<Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>>,
done: bool,
complete: bool,
}
impl ConfigStream {
pub fn new(inp: ChannelNameStream, node: NodeConfigCached, conf: ChannelArchiver) -> Self {
Self {
node,
conf,
inp,
inp_done: false,
get_fut: None,
update_fut: None,
done: false,
complete: false,
}
}
}
impl Stream for ConfigStream {
type Item = Result<ConfigItem, Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.complete {
panic!("poll on complete")
} else if self.done {
self.complete = true;
Ready(None)
} else if let Some(fut) = &mut self.update_fut {
match fut.poll_unpin(cx) {
Ready(Ok(_)) => {
self.update_fut = None;
continue;
}
Ready(Err(e)) => {
self.update_fut = None;
self.done = true;
Ready(Some(Err(e)))
}
Pending => Pending,
}
} else if let Some(fut) = &mut self.get_fut {
match fut.poll_unpin(cx) {
Ready(Ok(Res::Response(item))) => {
self.get_fut = None;
let name = item.channel.name.clone();
let dbconf = self.node.node_config.cluster.database.clone();
let config = serde_json::to_value(&item)?;
let fut = async move {
let dbc = database_connect(&dbconf).await?;
dbc.query("update channels set config = $2 where name = $1", &[&name, &config])
.await
.errstr()?;
Ok(())
};
self.update_fut = Some(Box::pin(fut));
let item = ConfigItem::Config(item);
Ready(Some(Ok(item)))
}
Ready(Ok(Res::TimedOut(name))) => {
self.get_fut = None;
let dbconf = self.node.node_config.cluster.database.clone();
let config = serde_json::to_value(&"TimedOut")?;
let fut = async move {
let dbc = database_connect(&dbconf).await?;
dbc.query("update channels set config = $2 where name = $1", &[&name, &config])
.await
.errstr()?;
Ok(())
};
self.update_fut = Some(Box::pin(fut));
continue;
}
Ready(Err(e)) => {
self.get_fut = None;
self.done = true;
Ready(Some(Err(e)))
}
Pending => Pending,
}
} else {
if self.inp_done {
self.done = true;
continue;
} else {
match self.inp.poll_next_unpin(cx) {
Ready(Some(Ok(item))) => {
let conf = self.conf.clone();
let database = self.node.node_config.cluster.database.clone();
let fut = async move {
let channel = Channel {
name: item,
backend: "".into(),
series: None,
};
let now = SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let beg = now - 60 * 60 * 1000;
let end = now + 60 * 60 * 4;
let q = ChannelConfigQuery {
channel,
range: NanoRange { beg, end },
expand: true,
};
let fut = super::channel_config(&q, &conf, &database);
let fut = tokio::time::timeout(Duration::from_millis(2000), fut);
match fut.await {
Ok(Ok(k)) => Ok(Res::Response(k)),
Ok(Err(e)) => Err(e),
Err(_) => {
warn!("timeout");
Ok(Res::TimedOut(q.channel.name))
}
}
};
self.get_fut = Some(Box::pin(fut));
continue;
}
Ready(Some(Err(e))) => {
self.done = true;
Ready(Some(Err(e)))
}
Ready(None) => {
self.inp_done = true;
info!("ConfigStream input done.");
continue;
}
Pending => Pending,
}
}
};
}
}
}
archapp/src/archeng/datablock.rs
-543
View File
@@ -1,543 +0,0 @@
use crate::archeng::indextree::DataheaderPos;
use crate::archeng::{format_hex_block, read_string, readf64, readu16, readu32, StatsChannel, EPICS_EPOCH_OFFSET};
use commonio::ringbuf::RingBuf;
use commonio::{read_exact, seek};
use err::Error;
use items::eventsitem::EventsItem;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::scalarevents::ScalarEvents;
use items::waveevents::WaveEvents;
use netpod::log::*;
use netpod::timeunits::SEC;
use netpod::{NanoRange, Nanos};
use std::convert::TryInto;
use std::io::SeekFrom;
use tokio::fs::File;
#[derive(Clone, Debug)]
enum DbrType {
DbrString = 0,
DbrShort = 1,
DbrStsFloat = 9,
DbrTimeString = 14,
DbrTimeShort = 15,
DbrTimeFloat = 16,
DbrTimeEnum = 17,
DbrTimeChar = 18,
DbrTimeLong = 19,
DbrTimeDouble = 20,
}
impl DbrType {
fn from_u16(k: u16) -> Result<Self, Error> {
use DbrType::*;
let res = match k {
0 => DbrString,
1 => DbrShort,
9 => DbrStsFloat,
14 => DbrTimeString,
15 => DbrTimeShort,
16 => DbrTimeFloat,
17 => DbrTimeEnum,
18 => DbrTimeChar,
19 => DbrTimeLong,
20 => DbrTimeDouble,
_ => {
let msg = format!("not a valid/supported dbr type: {}", k);
return Err(Error::with_msg_no_trace(msg));
}
};
Ok(res)
}
fn meta_len(&self) -> usize {
use DbrType::*;
match self {
DbrString => 0,
DbrShort => 0,
DbrStsFloat => 4,
DbrTimeString => 12,
DbrTimeShort => 12,
DbrTimeFloat => 12,
DbrTimeEnum => 12,
DbrTimeChar => 12,
DbrTimeLong => 12,
DbrTimeDouble => 12,
}
}
fn pad_meta(&self) -> usize {
use DbrType::*;
match self {
DbrString => 0,
DbrShort => 0,
DbrStsFloat => 0,
DbrTimeString => 0,
DbrTimeShort => 2,
DbrTimeFloat => 0,
DbrTimeEnum => 2,
DbrTimeChar => 3,
DbrTimeLong => 0,
DbrTimeDouble => 4,
}
}
fn val_len(&self) -> usize {
use DbrType::*;
match self {
DbrString => 40,
DbrShort => 2,
DbrStsFloat => 4,
DbrTimeString => 40,
DbrTimeShort => 2,
DbrTimeFloat => 4,
DbrTimeEnum => 2,
DbrTimeChar => 1,
DbrTimeLong => 4,
DbrTimeDouble => 8,
}
}
fn msg_len(&self, count: usize) -> usize {
let n = self.meta_len() + self.pad_meta() + count * self.val_len();
let r = n % 8;
let n = if r == 0 { n } else { n + 8 - r };
n
}
}
#[derive(Debug)]
pub struct DatafileHeader {
pos: DataheaderPos,
#[allow(unused)]
dir_offset: u32,
// Should be absolute file position of the next data header
// together with `fname_next`.
// But unfortunately not always set?
#[allow(unused)]
next_offset: u32,
#[allow(unused)]
prev_offset: u32,
#[allow(unused)]
curr_offset: u32,
pub num_samples: u32,
#[allow(unused)]
ctrl_info_offset: u32,
buf_size: u32,
#[allow(unused)]
buf_free: u32,
dbr_type: DbrType,
dbr_count: usize,
#[allow(unused)]
period: f64,
#[allow(unused)]
ts_beg: Nanos,
#[allow(unused)]
ts_end: Nanos,
#[allow(unused)]
ts_next_file: Nanos,
#[allow(unused)]
fname_next: String,
#[allow(unused)]
fname_prev: String,
}
const DATA_HEADER_LEN_ON_DISK: usize = 72 + 40 + 40;
// TODO retire this version (better version reads from buffer)
pub async fn read_datafile_header(
file: &mut File,
pos: DataheaderPos,
stats: &StatsChannel,
) -> Result<DatafileHeader, Error> {
let mut rb = RingBuf::new(file, pos.0, stats.clone()).await?;
rb.fill_min(DATA_HEADER_LEN_ON_DISK).await?;
let buf = rb.data();
let dir_offset = readu32(buf, 0);
let next_offset = readu32(buf, 4);
let prev_offset = readu32(buf, 8);
let curr_offset = readu32(buf, 12);
let num_samples = readu32(buf, 16);
let ctrl_info_offset = readu32(buf, 20);
let buf_size = readu32(buf, 24);
let buf_free = readu32(buf, 28);
let dbr_type = DbrType::from_u16(readu16(buf, 32))?;
let dbr_count = readu16(buf, 34);
// 4 bytes padding.
let period = readf64(buf, 40);
let ts1a = readu32(buf, 48);
let ts1b = readu32(buf, 52);
let ts2a = readu32(buf, 56);
let ts2b = readu32(buf, 60);
let ts3a = readu32(buf, 64);
let ts3b = readu32(buf, 68);
let ts_beg = if ts1a != 0 || ts1b != 0 {
ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let ts_end = if ts3a != 0 || ts3b != 0 {
ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let ts_next_file = if ts2a != 0 || ts2b != 0 {
ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let fname_prev = read_string(&buf[72..112])?;
let fname_next = read_string(&buf[112..152])?;
let ret = DatafileHeader {
pos,
dir_offset,
next_offset,
prev_offset,
curr_offset,
num_samples,
ctrl_info_offset,
buf_size,
buf_free,
dbr_type,
dbr_count: dbr_count as usize,
period,
ts_beg: Nanos { ns: ts_beg },
ts_end: Nanos { ns: ts_end },
ts_next_file: Nanos { ns: ts_next_file },
fname_next,
fname_prev,
};
Ok(ret)
}
pub async fn read_datafile_header2(rb: &mut RingBuf<File>, pos: DataheaderPos) -> Result<DatafileHeader, Error> {
// TODO avoid the extra seek: make sure that RingBuf catches this. Profile..
rb.seek(pos.0).await?;
rb.fill_min(DATA_HEADER_LEN_ON_DISK).await?;
let buf = rb.data();
let dir_offset = readu32(buf, 0);
let next_offset = readu32(buf, 4);
let prev_offset = readu32(buf, 8);
let curr_offset = readu32(buf, 12);
let num_samples = readu32(buf, 16);
let ctrl_info_offset = readu32(buf, 20);
let buf_size = readu32(buf, 24);
let buf_free = readu32(buf, 28);
let dbr_type = DbrType::from_u16(readu16(buf, 32))?;
let dbr_count = readu16(buf, 34);
// 4 bytes padding.
let period = readf64(buf, 40);
let ts1a = readu32(buf, 48);
let ts1b = readu32(buf, 52);
let ts2a = readu32(buf, 56);
let ts2b = readu32(buf, 60);
let ts3a = readu32(buf, 64);
let ts3b = readu32(buf, 68);
let ts_beg = if ts1a != 0 || ts1b != 0 {
ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let ts_end = if ts3a != 0 || ts3b != 0 {
ts3a as u64 * SEC + ts3b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let ts_next_file = if ts2a != 0 || ts2b != 0 {
ts2a as u64 * SEC + ts2b as u64 + EPICS_EPOCH_OFFSET
} else {
0
};
let fname_prev = read_string(&buf[72..112])?;
let fname_next = read_string(&buf[112..152])?;
rb.adv(DATA_HEADER_LEN_ON_DISK);
let ret = DatafileHeader {
pos,
dir_offset,
next_offset,
prev_offset,
curr_offset,
num_samples,
ctrl_info_offset,
buf_size,
buf_free,
dbr_type,
dbr_count: dbr_count as usize,
period,
ts_beg: Nanos { ns: ts_beg },
ts_end: Nanos { ns: ts_end },
ts_next_file: Nanos { ns: ts_next_file },
fname_next,
fname_prev,
};
Ok(ret)
}
trait MetaParse {
fn parse_meta(buf: &[u8]) -> (u64, usize);
}
struct NoneMetaParse;
impl MetaParse for NoneMetaParse {
#[inline(always)]
fn parse_meta(_buf: &[u8]) -> (u64, usize) {
(0, 0)
}
}
struct TimeMetaParse;
impl MetaParse for TimeMetaParse {
#[inline(always)]
fn parse_meta(buf: &[u8]) -> (u64, usize) {
let tsa = u32::from_be_bytes(buf[4..8].try_into().unwrap());
let tsb = u32::from_be_bytes(buf[8..12].try_into().unwrap());
let ts = tsa as u64 * SEC + tsb as u64 + EPICS_EPOCH_OFFSET;
(ts, 12)
}
}
#[inline(always)]
fn parse_msg<MP: MetaParse, F, VT>(
buf: &[u8],
_meta_parse: MP,
dbrt: DbrType,
dbrcount: usize,
valf: F,
) -> Result<(u64, VT, usize), Error>
where
F: Fn(&[u8], usize) -> VT,
{
let (ts, n) = MP::parse_meta(buf);
let buf = &buf[n + dbrt.pad_meta()..];
Ok((ts, valf(buf, dbrcount), n))
}
macro_rules! ex_s {
($sty:ident, $n:ident) => {
fn $n(buf: &[u8], _dbrcount: usize) -> $sty {
const R: usize = std::mem::size_of::<$sty>();
$sty::from_be_bytes(buf[0..R].try_into().unwrap())
}
};
}
macro_rules! ex_v {
($sty:ident, $n:ident) => {
fn $n(mut buf: &[u8], dbrcount: usize) -> Vec<$sty> {
const R: usize = std::mem::size_of::<$sty>();
let mut a = Vec::with_capacity(dbrcount);
for _ in 0..dbrcount {
let v = $sty::from_be_bytes(buf[0..R].try_into().unwrap());
a.push(v);
buf = &buf[R..];
}
a
}
};
}
ex_s!(i8, ex_s_i8);
ex_s!(i16, ex_s_i16);
ex_s!(i32, ex_s_i32);
ex_s!(f32, ex_s_f32);
ex_s!(f64, ex_s_f64);
ex_v!(i8, ex_v_i8);
ex_v!(i16, ex_v_i16);
ex_v!(i32, ex_v_i32);
ex_v!(f32, ex_v_f32);
ex_v!(f64, ex_v_f64);
macro_rules! read_msg {
($sty:ident, $exfs:ident, $exfv:ident, $evvar:ident, $rb:expr, $msglen:expr, $numsamples:expr, $dbrt:expr, $dbrcount:ident) => {
if $dbrcount == 1 {
let mut evs = ScalarEvents::empty();
for _ in 0..$numsamples {
$rb.fill_min($msglen).await?;
let buf = $rb.data();
let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfs)?;
evs.tss.push(ts);
evs.values.push(val);
$rb.adv($msglen);
}
let evs = ScalarPlainEvents::$evvar(evs);
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
let mut evs = WaveEvents::empty();
for _ in 0..$numsamples {
$rb.fill_min($msglen).await?;
let buf = $rb.data();
let (ts, val, _) = parse_msg(buf, TimeMetaParse, $dbrt.clone(), $dbrcount, $exfv)?;
evs.tss.push(ts);
evs.vals.push(val);
$rb.adv($msglen);
}
let evs = WavePlainEvents::$evvar(evs);
let plain = PlainEvents::Wave(evs);
let item = EventsItem::Plain(plain);
item
}
};
}
async fn _format_debug_1(rb: &mut RingBuf<File>, dbrcount: usize) -> Result<(), Error> {
rb.fill_min(1024 * 10).await?;
for i1 in 0..19 {
let hex = format_hex_block(&rb.data()[512 * i1..], 512);
error!("dbrcount {} block\n{}", dbrcount, hex);
}
return Err(Error::with_msg_no_trace("EXIT"));
}
fn _format_debug_2(evs: WaveEvents<i32>) -> Result<(), Error> {
info!("tss: {:?}", evs.tss);
let n = evs.vals.len();
let vals: Vec<_> = evs
.vals
.iter()
.enumerate()
.filter(|&(i, _)| i < 3 || i + 3 >= n)
.map(|(_i, j)| {
if j.len() > 6 {
let mut a = j[0..3].to_vec();
a.extend_from_slice(&j[j.len() - 3..]);
a.to_vec()
} else {
j.to_vec()
}
})
.collect();
info!("vals: {:?}", vals);
Ok(())
}
pub async fn read_data2(
rb: &mut RingBuf<File>,
datafile_header: &DatafileHeader,
_range: NanoRange,
_expand: bool,
) -> Result<EventsItem, Error> {
// TODO handle range
// TODO handle expand mode
{
let dpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64;
if rb.rp_abs() != dpos {
warn!("read_data2 rb not positioned {} vs {}", rb.rp_abs(), dpos);
rb.seek(dpos).await?;
}
}
let numsamples = datafile_header.num_samples as usize;
let dbrcount = datafile_header.dbr_count;
let dbrt = datafile_header.dbr_type.clone();
let dbrt = if let DbrType::DbrTimeEnum = dbrt {
DbrType::DbrTimeShort
} else {
dbrt
};
let msg_len = dbrt.msg_len(dbrcount);
{
if (datafile_header.buf_size as usize) < numsamples * msg_len {
return Err(Error::with_msg_no_trace(format!(
"buffer too small for data {} {} {}",
datafile_header.buf_size, numsamples, msg_len
)));
}
}
if dbrcount == 0 {
return Err(Error::with_msg_no_trace(format!("unexpected dbrcount {}", dbrcount)));
}
let res = match &dbrt {
DbrType::DbrTimeChar => read_msg!(i8, ex_s_i8, ex_v_i8, I8, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeShort => read_msg!(i16, ex_s_i16, ex_v_i16, I16, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeLong => read_msg!(i32, ex_s_i32, ex_v_i32, I32, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeFloat => read_msg!(f32, ex_s_f32, ex_v_f32, F32, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeDouble => read_msg!(f64, ex_s_f64, ex_v_f64, F64, rb, msg_len, numsamples, dbrt, dbrcount),
DbrType::DbrTimeString => {
if dbrcount == 1 {
// TODO
let evs = ScalarPlainEvents::I8(ScalarEvents::empty());
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
// TODO
let evs = WavePlainEvents::F64(WaveEvents::empty());
let plain = PlainEvents::Wave(evs);
let item = EventsItem::Plain(plain);
item
}
}
DbrType::DbrTimeEnum | DbrType::DbrShort | DbrType::DbrString | DbrType::DbrStsFloat => {
let msg = format!("Type {:?} not yet supported", datafile_header.dbr_type);
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));
}
};
Ok(res)
}
pub async fn read_data_1(
file: &mut File,
datafile_header: &DatafileHeader,
range: NanoRange,
_expand: bool,
stats: &StatsChannel,
) -> Result<EventsItem, Error> {
// TODO handle expand mode
let dhpos = datafile_header.pos.0 + DATA_HEADER_LEN_ON_DISK as u64;
seek(file, SeekFrom::Start(dhpos), stats).await?;
let res = match &datafile_header.dbr_type {
DbrType::DbrTimeDouble => {
if datafile_header.dbr_count == 1 {
trace!("~~~~~~~~~~~~~~~~~~~~~ read scalar DbrTimeDouble");
let mut evs = ScalarEvents::empty();
let n1 = datafile_header.num_samples as usize;
//let n2 = datafile_header.dbr_type.byte_len();
let n2 = 2 + 2 + 4 + 4 + (4) + 8;
let n3 = n1 * n2;
let mut buf = vec![0; n3];
read_exact(file, &mut buf, stats).await?;
let mut p1 = 0;
let mut ntot = 0;
while p1 < n3 - n2 {
let _status = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap());
p1 += 2;
let _severity = u16::from_be_bytes(buf[p1..p1 + 2].try_into().unwrap());
p1 += 2;
let ts1a = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap());
p1 += 4;
let ts1b = u32::from_be_bytes(buf[p1..p1 + 4].try_into().unwrap());
p1 += 4;
let ts1 = ts1a as u64 * SEC + ts1b as u64 + EPICS_EPOCH_OFFSET;
p1 += 4;
let value = f64::from_be_bytes(buf[p1..p1 + 8].try_into().unwrap());
p1 += 8;
ntot += 1;
if ts1 >= range.beg && ts1 < range.end {
evs.tss.push(ts1);
evs.values.push(value);
}
}
debug!("parsed block with {} / {} events", ntot, evs.tss.len());
let evs = ScalarPlainEvents::F64(evs);
let plain = PlainEvents::Scalar(evs);
let item = EventsItem::Plain(plain);
item
} else {
let msg = format!("dbr_count {:?} not yet supported", datafile_header.dbr_count);
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));
}
}
_ => {
let msg = format!("Type {:?} not yet supported", datafile_header.dbr_type);
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));
}
};
Ok(res)
}
archapp/src/archeng/diskio.rs
View File
archapp/src/archeng/indexfiles.rs
-635
View File
@@ -1,635 +0,0 @@
use crate::timed::Timed;
use crate::wrap_task;
use async_channel::Receiver;
use commonio::{open_read, read, StatsChannel};
use err::{ErrStr, Error};
use futures_core::{Future, Stream};
use futures_util::stream::unfold;
use netpod::log::*;
use netpod::NodeConfigCached;
use netpod::{Channel, ChannelArchiver, Database};
use regex::Regex;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::fs::read_dir;
use tokio::sync::Mutex;
use tokio_postgres::Client as PgClient;
pub fn list_index_files(node: &ChannelArchiver) -> Receiver<Result<PathBuf, Error>> {
let node = node.clone();
let (tx, rx) = async_channel::bounded(4);
let tx2 = tx.clone();
let task = async move {
for bp in &node.data_base_paths {
let mut rd = read_dir(bp).await?;
while let Some(e) = rd.next_entry().await? {
let ft = e.file_type().await?;
if ft.is_dir() {
let mut rd = read_dir(e.path()).await?;
while let Some(e) = rd.next_entry().await? {
let ft = e.file_type().await?;
if false && ft.is_dir() {
let mut rd = read_dir(e.path()).await?;
while let Some(e) = rd.next_entry().await? {
let ft = e.file_type().await?;
if ft.is_file() {
if e.file_name().to_string_lossy() == "index" {
tx.send(Ok(e.path())).await.errstr()?;
}
}
}
} else if ft.is_file() {
if e.file_name().to_string_lossy() == "index" {
tx.send(Ok(e.path())).await.errstr()?;
}
}
}
} else if ft.is_file() {
if e.file_name().to_string_lossy() == "index" {
tx.send(Ok(e.path())).await.errstr()?;
}
}
}
}
Ok::<_, Error>(())
};
wrap_task(task, tx2);
rx
}
pub struct ScanIndexFiles0 {}
impl Stream for ScanIndexFiles0 {
type Item = ();
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
let _ = cx;
todo!()
}
}
pub async fn get_level_0(conf: ChannelArchiver) -> Result<Vec<PathBuf>, Error> {
let mut ret = vec![];
for bp in &conf.data_base_paths {
let mut rd = read_dir(bp).await?;
while let Some(e) = rd.next_entry().await? {
if e.file_name().to_string_lossy().contains("index") {
warn!("Top-level data path contains `index` entry");
}
let ft = e.file_type().await?;
if ft.is_dir() {
ret.push(e.path());
}
}
}
Ok(ret)
}
pub async fn get_level_1(lev0: Vec<PathBuf>) -> Result<Vec<PathBuf>, Error> {
let mut ret = vec![];
for bp in lev0 {
let mut rd = read_dir(bp).await?;
while let Some(e) = rd.next_entry().await? {
let ft = e.file_type().await?;
if ft.is_file() {
if e.file_name().to_string_lossy() == "index" {
ret.push(e.path());
}
}
}
}
Ok(ret)
}
pub async fn database_connect(db_config: &Database) -> Result<PgClient, Error> {
let d = db_config;
let dbport = 5432;
let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, dbport, d.name);
let (cl, conn) = tokio_postgres::connect(&uri, tokio_postgres::NoTls)
.await
.map_err(|e| {
error!(
"Can not connect to database postgresql://{}:...@{}:{}/{}",
d.user, d.host, dbport, d.name
);
e
})
.errstr()?;
// TODO monitor connection drop.
let _cjh = tokio::spawn(async move {
if let Err(e) = conn.await {
error!("connection error: {}", e);
}
Ok::<_, Error>(())
});
Ok(cl)
}
pub trait UnfoldExec {
type Output: Send;
fn exec(self) -> Pin<Box<dyn Future<Output = Result<Option<(Self::Output, Self)>, Error>> + Send>>
where
Self: Sized;
}
pub fn unfold_stream<St, T>(st: St) -> impl Stream<Item = Result<T, Error>>
where
St: UnfoldExec<Output = T> + Send,
T: Send,
{
enum UnfoldState<St> {
Running(St),
Done,
}
unfold(UnfoldState::Running(st), |st| async move {
match st {
UnfoldState::Running(st) => match st.exec().await {
Ok(Some((item, st))) => Some((Ok(item), UnfoldState::Running(st))),
Ok(None) => None,
Err(e) => Some((Err(e), UnfoldState::Done)),
},
UnfoldState::Done => None,
}
})
}
enum ScanIndexFilesSteps {
Level0,
Level1(Vec<PathBuf>),
Done,
}
struct ScanIndexFiles {
node: NodeConfigCached,
conf: ChannelArchiver,
steps: ScanIndexFilesSteps,
}
impl ScanIndexFiles {
fn new(conf: ChannelArchiver, node: NodeConfigCached) -> Self {
Self {
node,
conf,
steps: ScanIndexFilesSteps::Level0,
}
}
async fn exec(mut self) -> Result<Option<(String, Self)>, Error> {
match self.steps {
ScanIndexFilesSteps::Level0 => {
let res = get_level_0(self.conf.clone()).await?;
self.steps = ScanIndexFilesSteps::Level1(res);
let item = format!("level 0 done");
Ok(Some((item, self)))
}
ScanIndexFilesSteps::Level1(paths) => {
let paths = get_level_1(paths).await?;
info!("collected {} level 1 paths", paths.len());
let dbc = database_connect(&self.node.node_config.cluster.database).await?;
for p in paths {
let ps = p.to_string_lossy();
let rows = dbc
.query("select rowid from indexfiles where path = $1", &[&ps])
.await
.errstr()?;
let rid: i64 = if rows.len() == 0 {
let rows = dbc
.query(
"insert into indexfiles (path) values ($1) on conflict do nothing returning rowid",
&[&ps],
)
.await
.errstr()?;
if rows.len() == 0 {
error!("insert failed, maybe concurrent insert?");
// TODO try this channel again? or the other process handled it?
err::todoval()
} else if rows.len() == 1 {
let rid = rows[0].try_get(0).errstr()?;
info!("insert done: {}", rid);
rid
} else {
return Err(Error::with_msg("not unique"));
}
} else if rows.len() == 1 {
let rid = rows[0].try_get(0).errstr()?;
rid
} else {
return Err(Error::with_msg("not unique"));
};
let _ = rid;
}
self.steps = ScanIndexFilesSteps::Done;
let item = format!("level 1 done");
Ok(Some((item, self)))
}
ScanIndexFilesSteps::Done => Ok(None),
}
}
}
impl UnfoldExec for ScanIndexFiles {
type Output = String;
fn exec(self) -> Pin<Box<dyn Future<Output = Result<Option<(Self::Output, Self)>, Error>> + Send>>
where
Self: Sized,
{
Box::pin(self.exec())
}
}
pub fn scan_index_files(conf: ChannelArchiver, node: NodeConfigCached) -> impl Stream<Item = Result<String, Error>> {
unfold_stream(ScanIndexFiles::new(conf.clone(), node))
/*
enum UnfoldState {
Running(ScanIndexFiles),
Done,
}
unfold(UnfoldState::Running(ScanIndexFiles::new(conf)), |st| async move {
match st {
UnfoldState::Running(st) => match st.exec().await {
Ok(Some((item, st))) => Some((Ok(item), UnfoldState::Running(st))),
Ok(None) => None,
Err(e) => {
error!("{}", e);
Some((Err(e), UnfoldState::Done))
}
},
UnfoldState::Done => None,
}
})
*/
}
pub fn unfold1() -> impl Stream<Item = String> {
unfold(123u32, |st| async move { Some((format!("{}", st), st)) })
}
pub fn unfold2(_conf: ChannelArchiver) -> () {
/*let f1 = async move {
let _list = get_level_0(conf).await?;
let yld = format!("level 0 done");
let fut = async { Ok(None) };
Ok(Some((yld, Box::pin(fut))))
};
unfold(
Box::pin(f1) as Pin<Box<dyn Future<Output = Result<Option<(String, _)>, Error>>>>,
|st| async {
match st.await {
Ok(None) => None,
Ok(Some((item, st))) => {
//Some((item, st));
//Some((String::new(), Box::pin(async { Ok(None) })))
None
}
Err(e) => {
error!("{}", e);
None
}
}
},
)*/
err::todoval()
}
// -------------------------------------------------
enum ScanChannelsSteps {
Start,
SelectIndexFile,
ReadChannels(Vec<String>),
Done,
}
struct ScanChannels {
node: NodeConfigCached,
#[allow(unused)]
conf: ChannelArchiver,
steps: ScanChannelsSteps,
}
impl ScanChannels {
fn new(node: NodeConfigCached, conf: ChannelArchiver) -> Self {
Self {
node,
conf,
steps: ScanChannelsSteps::Start,
}
}
async fn exec(mut self) -> Result<Option<(String, Self)>, Error> {
use ScanChannelsSteps::*;
match self.steps {
Start => {
self.steps = SelectIndexFile;
Ok(Some((format!("Start"), self)))
}
SelectIndexFile => {
let dbc = database_connect(&self.node.node_config.cluster.database).await?;
let sql =
"select path from indexfiles where ts_last_channel_search < now() - interval '1 hour' limit 1";
let rows = dbc.query(sql, &[]).await.errstr()?;
let mut paths = vec![];
for row in rows {
paths.push(row.get::<_, String>(0));
}
let item = format!("SelectIndexFile {:?}", paths);
self.steps = ReadChannels(paths);
Ok(Some((item, self)))
}
ReadChannels(mut paths) => {
// TODO stats
let stats = &StatsChannel::dummy();
let dbc = database_connect(&self.node.node_config.cluster.database).await?;
if let Some(path) = paths.pop() {
let rows = dbc
.query("select rowid from indexfiles where path = $1", &[&path])
.await
.errstr()?;
if rows.len() == 1 {
let indexfile_rid: i64 = rows[0].try_get(0).errstr()?;
let mut file = open_read(path.clone().into(), stats).await?;
let mut basics = super::indextree::IndexFileBasics::from_file(path, &mut file, stats).await?;
let entries = basics.all_channel_entries(&mut file, stats).await?;
for entry in entries {
let rows = dbc
.query("select rowid from channels where name = $1", &[&entry.channel_name()])
.await
.errstr()?;
let rid: i64 = if rows.len() == 0 {
let rows = dbc
.query(
"insert into channels (name) values ($1) on conflict do nothing returning rowid",
&[&entry.channel_name()],
)
.await.errstr()?;
if rows.len() == 0 {
error!("insert failed, maybe concurrent insert?");
// TODO try this channel again? or the other process handled it?
err::todoval()
} else if rows.len() == 1 {
let rid = rows[0].try_get(0).errstr()?;
info!("insert done: {}", rid);
rid
} else {
return Err(Error::with_msg("not unique"));
}
} else if rows.len() == 1 {
let rid = rows[0].try_get(0).errstr()?;
rid
} else {
return Err(Error::with_msg("not unique"));
};
dbc.query(
"insert into channel_index_map (channel, index) values ($1, $2) on conflict do nothing",
&[&rid, &indexfile_rid],
)
.await
.errstr()?;
}
dbc.query(
"update indexfiles set ts_last_channel_search = now() where rowid = $1",
&[&indexfile_rid],
)
.await
.errstr()?;
}
}
self.steps = Done;
Ok(Some((format!("ReadChannels"), self)))
}
Done => Ok(None),
}
}
}
impl UnfoldExec for ScanChannels {
type Output = String;
fn exec(self) -> Pin<Box<dyn Future<Output = Result<Option<(Self::Output, Self)>, Error>> + Send>>
where
Self: Sized,
{
Box::pin(self.exec())
}
}
pub fn scan_channels(node: NodeConfigCached, conf: ChannelArchiver) -> impl Stream<Item = Result<String, Error>> {
unfold_stream(ScanChannels::new(node, conf.clone()))
}
#[derive(Debug)]
enum RetClass {
Long,
Medium,
Short,
#[allow(unused)]
PostMortem,
}
#[derive(Debug)]
enum IndexCat {
Machine {
rc: RetClass,
},
#[allow(unused)]
Beamline {
rc: RetClass,
name: String,
},
}
#[derive(Debug)]
struct IndexFile {
path: PathBuf,
cat: IndexCat,
}
// Try to make sense of historical conventions how the epics channel archiver engines are configured.
fn categorize_index_files(list: &Vec<String>) -> Result<Vec<IndexFile>, Error> {
let re_m = Regex::new(r"/archive_(ST|MT|LT)/index").unwrap();
let re_b = Regex::new(r"/archive_(X([0-9]+)[^_]*)_(SH|LO)/index").unwrap();
let mut ret = vec![];
for p in list {
match re_m.captures(p) {
Some(cap) => {
let rc = cap.get(1).unwrap().as_str();
let rc = match rc {
"ST" => Some(RetClass::Short),
"MT" => Some(RetClass::Medium),
"LT" => Some(RetClass::Long),
_ => {
warn!("categorize_index_files no idea about RC for {}", p);
None
}
};
if let Some(rc) = rc {
let f = IndexFile {
path: p.into(),
cat: IndexCat::Machine { rc },
};
ret.push(f);
}
}
None => match re_b.captures(p) {
Some(cap) => {
let name = cap.get(1).unwrap().as_str();
let rc = cap.get(3).unwrap().as_str();
let rc = match rc {
"SH" => Some(RetClass::Short),
"LO" => Some(RetClass::Long),
_ => {
warn!("categorize_index_files no idea about RC for {}", p);
None
}
};
if let Some(rc) = rc {
let f = IndexFile {
path: p.into(),
cat: IndexCat::Beamline { name: name.into(), rc },
};
ret.push(f);
}
}
None => {
warn!("categorize_index_files no idea at all about {}", p);
}
},
}
}
let is_machine = {
let mut k = false;
for x in &ret {
if let IndexCat::Machine { .. } = &x.cat {
k = true;
break;
}
}
k
};
// TODO by default, filter post-mortem.
let is_beamline = !is_machine;
if is_beamline {
let mut ret: Vec<_> = ret
.into_iter()
.filter_map(|k| {
if let IndexCat::Machine { rc, .. } = &k.cat {
let prio = match rc {
&RetClass::Short => 4,
&RetClass::Medium => 6,
&RetClass::Long => 8,
&RetClass::PostMortem => 0,
};
Some((k, prio))
} else {
None
}
})
.collect();
ret.sort_by_key(|x| x.1);
let ret = ret.into_iter().map(|k| k.0).collect();
Ok(ret)
} else if is_machine {
let mut ret: Vec<_> = ret
.into_iter()
.filter_map(|k| {
if let IndexCat::Machine { rc, .. } = &k.cat {
let prio = match rc {
&RetClass::Short => 4,
&RetClass::Medium => 6,
&RetClass::Long => 8,
&RetClass::PostMortem => 0,
};
Some((k, prio))
} else {
None
}
})
.collect();
ret.sort_by_key(|x| x.1);
let ret = ret.into_iter().map(|k| k.0).collect();
Ok(ret)
} else {
err::todoval()
}
}
pub async fn index_file_path_list(channel: Channel, dbconf: Database) -> Result<Vec<PathBuf>, Error> {
let dbc = database_connect(&dbconf).await.map_err(|e| {
error!("CAN NOT CONNECT TO DATABASE [{e:?}]");
e
})?;
let sql = "select i.path from indexfiles i, channels c, channel_index_map m where c.name = $1 and m.channel = c.rowid and i.rowid = m.index";
let rows = dbc.query(sql, &[&channel.name()]).await.errstr()?;
let mut index_paths = vec![];
for row in rows {
index_paths.push(row.try_get(0).errstr()?);
}
let list = categorize_index_files(&index_paths)?;
let ret = list.into_iter().map(|k| k.path).collect();
Ok(ret)
}
static INDEX_JSON: Mutex<Option<BTreeMap<String, Vec<String>>>> = Mutex::const_new(None);
async fn index_files_index_ref<P: Into<PathBuf> + Send>(
key: &str,
index_files_index_path: P,
stats: &StatsChannel,
) -> Result<Option<Vec<String>>, Error> {
let mut g = INDEX_JSON.lock().await;
match &*g {
Some(j) => Ok(j.get(key).map(|x| x.clone())),
None => {
let timed1 = Timed::new("slurp_index_json");
let index_files_index_path = index_files_index_path.into();
let index_files_index = {
let timed1 = Timed::new("slurp_index_bytes");
let mut index_files_index = match open_read(index_files_index_path.clone(), stats).await {
Ok(k) => Ok(k),
Err(e) => {
warn!("can not open indexfile {:?}", index_files_index_path);
Err(e)
}
}?;
let mut buf = vec![0; 1024 * 1024 * 50];
let mut ntot = 0;
loop {
let n = read(&mut index_files_index, &mut buf[ntot..], stats).await?;
if n == 0 {
break;
}
ntot += n;
}
buf.truncate(ntot);
drop(timed1);
serde_json::from_slice::<BTreeMap<String, Vec<String>>>(&buf)?
};
drop(timed1);
let ret = index_files_index.get(key).map(|x| x.clone());
*g = Some(index_files_index);
Ok(ret)
}
}
}
// TODO using the json index is currently no longer needed, but maybe as alternative for tests.
#[allow(unused)]
async fn index_file_path_list_old(
channel: Channel,
index_files_index_path: PathBuf,
stats: &StatsChannel,
) -> Result<Vec<PathBuf>, Error> {
let timed1 = Timed::new("categorize index files");
let index_paths = index_files_index_ref(channel.name(), &index_files_index_path, stats)
.await?
.ok_or(Error::with_msg_no_trace("can not find channel"))?;
let list = categorize_index_files(&index_paths)?;
info!("categorized:\n{:?}", list);
let ret = list.into_iter().map(|k| k.path).collect();
drop(timed1);
Ok(ret)
}
archapp/src/archeng/indextree.rs
-1406
View File
File diff suppressed because it is too large Load Diff
archapp/src/archeng/pipe.rs
-313
View File
@@ -1,313 +0,0 @@
use crate::archeng::blockrefstream::blockref_stream;
use crate::archeng::blockstream::BlockStream;
use crate::events::{FrameMaker, FrameMakerTrait};
use err::Error;
use futures_util::{Stream, StreamExt};
use items::binnedevents::{MultiBinWaveEvents, SingleBinWaveEvents, XBinnedEvents};
use items::eventsitem::EventsItem;
use items::plainevents::{PlainEvents, WavePlainEvents};
use items::waveevents::{WaveNBinner, WaveXBinner};
use items::{EventsNodeProcessor, Framable, LogItem, RangeCompletableItem, StreamItem};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::{AggKind, NodeConfigCached, Shape};
use netpod::{ChannelArchiver, ChannelConfigQuery};
use std::pin::Pin;
use streams::rangefilter::RangeFilter;
pub async fn make_event_pipe(
evq: &RawEventsQuery,
node: NodeConfigCached,
conf: ChannelArchiver,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
debug!("make_event_pipe {:?}", evq);
let channel_config = {
let q = ChannelConfigQuery {
channel: evq.channel.clone(),
range: evq.range.clone(),
expand: evq.agg_kind.need_expand(),
};
crate::archeng::channel_config_from_db(&q, &conf, &node.node_config.cluster.database).await?
};
debug!("Channel config: {:?}", channel_config);
let ixpaths = crate::archeng::indexfiles::index_file_path_list(
evq.channel.clone(),
node.node_config.cluster.database.clone(),
)
.await?;
debug!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = if let Some(x) = ixpaths.first() {
x.clone()
} else {
return Err(Error::with_msg_no_trace("no index file for channel")
.mark_bad_request()
.add_public_msg(format!("No index file for {}", evq.channel.name)));
};
use crate::archeng::blockstream::BlockItem;
let refs = blockref_stream(
evq.channel.clone(),
evq.range.clone(),
evq.agg_kind.need_expand(),
ixpath.clone(),
);
let blocks = BlockStream::new(Box::pin(refs), evq.range.clone(), 1);
let blocks = blocks.map(|k| match k {
Ok(item) => match item {
BlockItem::EventsItem(item) => Ok(StreamItem::DataItem(RangeCompletableItem::Data(item))),
BlockItem::JsVal(jsval) => Ok(StreamItem::Log(LogItem::quick(Level::DEBUG, format!("{:?}", jsval)))),
},
Err(e) => Err(e),
});
let cfgshape = channel_config.shape.clone();
let q_agg_kind = evq.agg_kind.clone();
let filtered = RangeFilter::new(blocks, evq.range.clone(), evq.agg_kind.need_expand());
let xtrans = match channel_config.shape {
Shape::Scalar => match evq.agg_kind {
AggKind::Plain => Box::pin(filtered) as Pin<Box<dyn Stream<Item = _> + Send>>,
AggKind::TimeWeightedScalar | AggKind::DimXBins1 => {
let tr = filtered.map(|j| match j {
Ok(j) => match j {
StreamItem::DataItem(j) => match j {
RangeCompletableItem::RangeComplete => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
RangeCompletableItem::Data(j) => match j {
EventsItem::Plain(j) => match j {
PlainEvents::Scalar(j) => {
let item = XBinnedEvents::Scalar(j);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
PlainEvents::Wave(_) => panic!(),
},
EventsItem::XBinnedEvents(_) => panic!(),
},
},
StreamItem::Log(j) => Ok(StreamItem::Log(j)),
StreamItem::Stats(j) => Ok(StreamItem::Stats(j)),
},
Err(e) => Err(e),
});
Box::pin(tr) as _
}
AggKind::DimXBinsN(_) => err::todoval(),
AggKind::EventBlobs => err::todoval(),
AggKind::Stats1 => err::todoval(),
},
Shape::Wave(_n1) => match evq.agg_kind {
AggKind::Plain => Box::pin(filtered) as Pin<Box<dyn Stream<Item = _> + Send>>,
AggKind::TimeWeightedScalar | AggKind::DimXBins1 => {
let tr = filtered.map(move |j| match j {
Ok(j) => match j {
StreamItem::DataItem(j) => match j {
RangeCompletableItem::RangeComplete => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
RangeCompletableItem::Data(j) => match j {
EventsItem::Plain(j) => match j {
PlainEvents::Scalar(_) => {
warn!("EventsItem::Plain Scalar for {:?} {:?}", cfgshape, q_agg_kind);
panic!()
}
PlainEvents::Wave(j) => {
trace!("EventsItem::Plain Wave for {:?} {:?}", cfgshape, q_agg_kind);
items_proc::tycases1!(j, WavePlainEvents, (j), {
let binner =
WaveXBinner::<$ty>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::$id(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
})
/*match j {
WavePlainEvents::I8(j) => {
let binner =
WaveXBinner::<i8>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::I8(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::I16(j) => {
let binner =
WaveXBinner::<i16>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::I16(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::I32(j) => {
let binner =
WaveXBinner::<i32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::I32(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::F32(j) => {
let binner =
WaveXBinner::<f32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::F32(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::F64(j) => {
let binner =
WaveXBinner::<f64>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = SingleBinWaveEvents::F64(out);
let item = XBinnedEvents::SingleBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
}*/
}
},
EventsItem::XBinnedEvents(j) => match j {
XBinnedEvents::Scalar(j) => {
warn!("XBinnedEvents::Scalar for {:?} {:?}", cfgshape, q_agg_kind);
let item = XBinnedEvents::Scalar(j);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
XBinnedEvents::SingleBinWave(j) => {
warn!("XBinnedEvents::SingleBinWave for {:?} {:?}", cfgshape, q_agg_kind);
let item = XBinnedEvents::SingleBinWave(j);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
XBinnedEvents::MultiBinWave(_) => todo!(),
},
},
},
StreamItem::Log(j) => Ok(StreamItem::Log(j)),
StreamItem::Stats(j) => Ok(StreamItem::Stats(j)),
},
Err(e) => Err(e),
});
Box::pin(tr) as _
}
AggKind::DimXBinsN(_) => {
let tr = filtered.map(move |j| match j {
Ok(j) => match j {
StreamItem::DataItem(j) => match j {
RangeCompletableItem::RangeComplete => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
RangeCompletableItem::Data(j) => match j {
EventsItem::Plain(j) => match j {
PlainEvents::Scalar(_) => {
warn!("EventsItem::Plain Scalar for {:?} {:?}", cfgshape, q_agg_kind);
panic!()
}
PlainEvents::Wave(j) => {
trace!("EventsItem::Plain Wave for {:?} {:?}", cfgshape, q_agg_kind);
items_proc::tycases1!(j, WavePlainEvents, (j), {
let binner =
WaveNBinner::<$ty>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::$id(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
})
/*match j {
WavePlainEvents::I8(j) => {
let binner =
WaveNBinner::<i8>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::I8(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::I16(j) => {
let binner =
WaveNBinner::<i16>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::I16(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::I32(j) => {
let binner =
WaveNBinner::<i32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::I32(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::F32(j) => {
let binner =
WaveNBinner::<f32>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::F32(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
WavePlainEvents::F64(j) => {
let binner =
WaveNBinner::<f64>::create(cfgshape.clone(), q_agg_kind.clone());
let out = binner.process(j);
let item = MultiBinWaveEvents::F64(out);
let item = XBinnedEvents::MultiBinWave(item);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
}*/
}
},
EventsItem::XBinnedEvents(j) => match j {
XBinnedEvents::Scalar(j) => {
warn!("XBinnedEvents::Scalar for {:?} {:?}", cfgshape, q_agg_kind);
err::todo();
let item = XBinnedEvents::Scalar(j);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
XBinnedEvents::SingleBinWave(j) => {
warn!("XBinnedEvents::SingleBinWave for {:?} {:?}", cfgshape, q_agg_kind);
err::todo();
let item = XBinnedEvents::SingleBinWave(j);
let item = EventsItem::XBinnedEvents(item);
Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))
}
XBinnedEvents::MultiBinWave(_) => todo!(),
},
},
},
StreamItem::Log(j) => Ok(StreamItem::Log(j)),
StreamItem::Stats(j) => Ok(StreamItem::Stats(j)),
},
Err(e) => Err(e),
});
Box::pin(tr) as _
}
AggKind::EventBlobs => err::todoval(),
AggKind::Stats1 => err::todoval(),
},
_ => {
error!("TODO shape {:?}", channel_config.shape);
let err = Error::with_msg_no_trace(format!("TODO shape {:?}", channel_config.shape))
.mark_bad_request()
.add_public_msg(format!("can not yet handle shape {:?}", channel_config.shape));
Box::pin(futures_util::stream::iter([Err(err)]))
}
};
let mut frame_maker = Box::new(FrameMaker::with_item_type(
channel_config.scalar_type.clone(),
channel_config.shape.clone(),
evq.agg_kind.clone(),
)) as Box<dyn FrameMakerTrait>;
let ret = xtrans.map(move |j| frame_maker.make_frame(j));
Ok(Box::pin(ret))
}
archapp/src/err.rs
-75
View File
@@ -1,75 +0,0 @@
use std::fmt;
pub struct ArchError(::err::Error);
impl ArchError {
pub fn with_msg<S: Into<String>>(s: S) -> Self {
Self(::err::Error::with_msg(s))
}
pub fn with_msg_no_trace<S: Into<String>>(s: S) -> Self {
Self(::err::Error::with_msg_no_trace(s))
}
pub fn msg(&self) -> &str {
self.0.msg()
}
pub fn reason(&self) -> Option<::err::Reason> {
self.0.reason()
}
pub fn public_msg(&self) -> Option<&Vec<String>> {
self.0.public_msg()
}
}
impl fmt::Debug for ArchError {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(fmt)
}
}
impl fmt::Display for ArchError {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(self, fmt)
}
}
impl std::error::Error for ArchError {}
impl From<::err::Error> for ArchError {
fn from(x: ::err::Error) -> Self {
Self(x)
}
}
impl From<ArchError> for ::err::Error {
fn from(x: ArchError) -> Self {
x.0
}
}
impl From<std::string::FromUtf8Error> for ArchError {
fn from(k: std::string::FromUtf8Error) -> Self {
Self::with_msg(k.to_string())
}
}
impl From<std::io::Error> for ArchError {
fn from(k: std::io::Error) -> Self {
Self::with_msg(k.to_string())
}
}
impl<T> From<async_channel::SendError<T>> for ArchError {
fn from(k: async_channel::SendError<T>) -> Self {
Self::with_msg(k.to_string())
}
}
impl From<serde_json::Error> for ArchError {
fn from(k: serde_json::Error) -> Self {
Self::with_msg(k.to_string())
}
}
archapp/src/events.rs
-761
View File
@@ -1,761 +0,0 @@
use crate::err::ArchError;
use crate::generated::EPICSEvent::PayloadType;
use crate::parse::multi::parse_all_ts;
use crate::parse::PbFileReader;
use crate::storagemerge::StorageMerge;
use chrono::{TimeZone, Utc};
use err::{ErrStr, Error};
use futures_core::Stream;
use futures_util::StreamExt;
use items::binnedevents::{MultiBinWaveEvents, SingleBinWaveEvents, XBinnedEvents};
use items::eventsitem::EventsItem;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::scalarevents::ScalarEvents;
use items::waveevents::WaveEvents;
use items::xbinnedscalarevents::XBinnedScalarEvents;
use items::xbinnedwaveevents::XBinnedWaveEvents;
use items::{Framable, RangeCompletableItem, Sitemty, SitemtyFrameType, StreamItem, WithLen, WithTimestamps};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::timeunits::{DAY, SEC};
use netpod::{AggKind, ArchiverAppliance, Channel, ChannelInfo, HasScalarType, HasShape, NanoRange, ScalarType, Shape};
use serde::Serialize;
use serde_json::Value as JsonValue;
use std::io::SeekFrom;
use std::path::PathBuf;
use std::pin::Pin;
use tokio::fs::{read_dir, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt};
#[derive(Debug)]
pub struct DataFilename {
pub year: u32,
pub month: u32,
}
pub fn parse_data_filename(s: &str) -> Result<DataFilename, Error> {
if !s.ends_with(".pb") {
return Err(Error::with_msg_no_trace("not a .pb file"));
}
if s.len() < 12 {
return Err(Error::with_msg_no_trace("filename too short"));
}
let j = &s[s.len() - 11..];
if &j[0..1] != ":" {
return Err(Error::with_msg_no_trace("no colon"));
}
if &j[5..6] != "_" {
return Err(Error::with_msg_no_trace("no underscore"));
}
let year: u32 = j[1..5].parse()?;
let month: u32 = j[6..8].parse()?;
let ret = DataFilename { year, month };
Ok(ret)
}
// TODO do we need Send here?
pub trait FrameMakerTrait: Send {
fn make_frame(&mut self, ei: Sitemty<EventsItem>) -> Box<dyn Framable + Send>;
}
pub struct FrameMaker {
scalar_type: ScalarType,
shape: Shape,
agg_kind: AggKind,
}
impl FrameMaker {
#[allow(dead_code)]
fn make_frame_gen<T>(_item: Sitemty<EventsItem>) -> Box<dyn Framable>
where
T: SitemtyFrameType + Serialize + Send + 'static,
{
err::todoval()
}
pub fn with_item_type(scalar_type: ScalarType, shape: Shape, agg_kind: AggKind) -> Self {
Self {
scalar_type: scalar_type,
shape: shape,
agg_kind: agg_kind,
}
}
}
#[allow(unused_macros)]
macro_rules! events_item_to_sitemty {
($ei:expr, $t1:ident, $t2:ident, $t3:ident) => {{
let combo = format!("t1 {} t2 {} t3 {}", stringify!($t1), stringify!($t2), stringify!($t3));
let ret = match $ei {
Ok(k) => match k {
StreamItem::DataItem(k) => match k {
RangeCompletableItem::Data(k) => {
//
match k {
EventsItem::Plain(h) => {
//
match h {
PlainEvents::$t1(h) => {
//
match h {
$t2::$t3(h) => Ok(StreamItem::DataItem(RangeCompletableItem::Data(h))),
_ => {
warn!("case AA {}", combo);
panic!()
}
}
}
_ => {
warn!("case BB {}", combo);
panic!()
}
}
}
_ => {
warn!("case CC {}", combo);
panic!()
}
}
}
RangeCompletableItem::RangeComplete => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
},
StreamItem::Log(j) => Ok(StreamItem::Log(j)),
StreamItem::Stats(j) => Ok(StreamItem::Stats(j)),
},
Err(e) => Err(e),
};
Box::new(ret)
}};
}
macro_rules! arm2 {
($item:expr, $t1:ident, $t2:ident, $t3:ident, $t4:ident, $t5:ident, $sty1:ident, $sty2:ident) => {{
type T1 = $t1<$sty1>;
let combo = format!(
"t1 {} t2 {} t3 {} t4 {} t5 {} sty1 {} sty2 {}",
stringify!($t1),
stringify!($t2),
stringify!($t3),
stringify!($t4),
stringify!($t5),
stringify!($sty1),
stringify!($sty2)
);
let ret: Sitemty<T1> = match $item {
Ok(k) => match k {
StreamItem::DataItem(k) => match k {
RangeCompletableItem::RangeComplete => {
Ok(StreamItem::DataItem(RangeCompletableItem::RangeComplete))
}
RangeCompletableItem::Data(k) => match k {
EventsItem::$t2(k) => match k {
$t3::$t4(k) => match k {
$t5::$sty2(k) => {
//
Ok(StreamItem::DataItem(RangeCompletableItem::Data(k)))
}
_ => {
warn!("unclear what to do A {}", combo);
err::todoval()
}
},
_ => {
warn!("unclear what to do B {}", combo);
err::todoval()
}
},
_ => {
error!("unexpected arm2 case {}", combo);
err::todoval()
}
},
},
StreamItem::Log(k) => Ok(StreamItem::Log(k)),
StreamItem::Stats(k) => Ok(StreamItem::Stats(k)),
},
Err(e) => Err(e),
};
Box::new(ret) as Box<dyn Framable + Send>
}};
}
macro_rules! arm1 {
($item:expr, $sty1:ident, $sty2:ident, $shape:expr, $ak:expr) => {{
if let AggKind::Stats1 = $ak {
err::todo();
return arm2!(
$item,
ScalarEvents,
Plain,
PlainEvents,
Scalar,
ScalarPlainEvents,
$sty1,
$sty2
);
}
match $shape {
Shape::Scalar => match $ak {
AggKind::EventBlobs => {
warn!("arm1 unhandled AggKind::EventBlobs");
panic!()
}
AggKind::Plain => arm2!(
$item,
ScalarEvents,
Plain,
PlainEvents,
Scalar,
ScalarPlainEvents,
$sty1,
$sty2
),
AggKind::TimeWeightedScalar => arm2!(
$item,
ScalarEvents,
XBinnedEvents,
XBinnedEvents,
Scalar,
ScalarPlainEvents,
$sty1,
$sty2
),
AggKind::DimXBins1 => arm2!(
$item,
ScalarEvents,
XBinnedEvents,
XBinnedEvents,
Scalar,
ScalarPlainEvents,
$sty1,
$sty2
),
AggKind::DimXBinsN(_) => arm2!(
$item,
ScalarEvents,
XBinnedEvents,
XBinnedEvents,
Scalar,
ScalarPlainEvents,
$sty1,
$sty2
),
// Handled above..
AggKind::Stats1 => panic!(),
},
Shape::Wave(_) => match $ak {
AggKind::EventBlobs => {
warn!("arm1 unhandled EventBlobs");
panic!()
}
AggKind::Plain => arm2!(
$item,
WaveEvents,
Plain,
PlainEvents,
Wave,
WavePlainEvents,
$sty1,
$sty2
),
AggKind::TimeWeightedScalar => arm2!(
$item,
XBinnedScalarEvents,
XBinnedEvents,
XBinnedEvents,
SingleBinWave,
SingleBinWaveEvents,
$sty1,
$sty2
),
AggKind::DimXBins1 => arm2!(
$item,
XBinnedScalarEvents,
XBinnedEvents,
XBinnedEvents,
SingleBinWave,
SingleBinWaveEvents,
$sty1,
$sty2
),
AggKind::DimXBinsN(_) => arm2!(
$item,
XBinnedWaveEvents,
XBinnedEvents,
XBinnedEvents,
MultiBinWave,
MultiBinWaveEvents,
$sty1,
$sty2
),
// Handled above..
AggKind::Stats1 => panic!(),
},
Shape::Image(..) => {
// There should be no images on archiver.
warn!("TODO for {:?}", $shape);
err::todoval()
}
}
}};
}
impl FrameMakerTrait for FrameMaker {
fn make_frame(&mut self, item: Sitemty<EventsItem>) -> Box<dyn Framable + Send> {
let scalar_type = &self.scalar_type;
let shape = &self.shape;
let agg_kind = &self.agg_kind;
match scalar_type {
ScalarType::I8 => arm1!(item, i8, I8, shape, agg_kind),
ScalarType::I16 => arm1!(item, i16, I16, shape, agg_kind),
ScalarType::I32 => arm1!(item, i32, I32, shape, agg_kind),
ScalarType::F32 => arm1!(item, f32, F32, shape, agg_kind),
ScalarType::F64 => arm1!(item, f64, F64, shape, agg_kind),
_ => {
warn!("TODO for scalar_type {:?}", scalar_type);
err::todoval()
}
}
}
}
pub async fn make_event_pipe(
evq: &RawEventsQuery,
aa: &ArchiverAppliance,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
let ci = channel_info(&evq.channel, aa).await?;
let mut inps = vec![];
let mut names = vec![];
for p1 in &aa.data_base_paths {
let p2 = p1.clone();
let p3 = make_single_event_pipe(evq, p2).await?;
inps.push(p3);
names.push(p1.to_str().unwrap().into());
}
let sm = StorageMerge::new(inps, names, evq.range.clone());
let mut frame_maker = Box::new(FrameMaker::with_item_type(
ci.scalar_type.clone(),
ci.shape.clone(),
evq.agg_kind.clone(),
)) as Box<dyn FrameMakerTrait>;
let ret = sm.map(move |j| frame_maker.make_frame(j));
Ok(Box::pin(ret))
}
pub async fn make_single_event_pipe(
evq: &RawEventsQuery,
base_path: PathBuf,
) -> Result<Pin<Box<dyn Stream<Item = Sitemty<EventsItem>> + Send>>, Error> {
// TODO must apply the proper x-binning depending on the requested AggKind.
debug!("make_single_event_pipe {:?}", evq);
let evq = evq.clone();
let DirAndPrefix { dir, prefix } = directory_for_channel_files(&evq.channel, &base_path)?;
//let dtbeg = Utc.timestamp((evq.range.beg / 1000000000) as i64, (evq.range.beg % 1000000000) as u32);
let (tx, rx) = async_channel::bounded(16);
let block1 = async move {
debug!("start read of {:?}", dir);
// TODO first collect all matching filenames, then sort, then open files.
// TODO if dir does not exist, should notify client but not log as error.
let mut rd = match tokio::fs::read_dir(&dir).await {
Ok(k) => k,
Err(e) => match e.kind() {
std::io::ErrorKind::NotFound => {
warn!("does not exist: {:?}", dir);
return Ok(());
}
_ => return Err(e)?,
},
};
while let Some(de) = rd.next_entry().await? {
let s = de.file_name().to_string_lossy().into_owned();
if s.starts_with(&prefix) && s.ends_with(".pb") {
match parse_data_filename(&s) {
Ok(df) => {
debug!("parse went ok: {} {}", df.year, df.month);
let ts0 = Utc.ymd(df.year as i32, df.month, 1).and_hms(0, 0, 0);
let ts1 = ts0.timestamp() as u64 * SEC + ts0.timestamp_subsec_nanos() as u64;
debug!("file {} {}", ts1, ts1 + DAY * 27);
debug!("range {} {}", evq.range.beg, evq.range.end);
if evq.range.beg < ts1 + DAY * 27 && evq.range.end > ts1 {
debug!("•••••••••••••••••••••••••• file matches requested range");
let f1 = File::open(de.path()).await?;
info!("opened {:?}", de.path());
let mut z = position_file_for_evq(f1, evq.clone(), df.year).await?;
let mut pbr = if let PositionState::Positioned(pos) = z.state {
z.pbr.reset_io(pos).await?;
z.pbr
} else {
continue;
};
let mut i1 = 0;
'evread: loop {
match pbr.read_msg().await {
Ok(Some(ei)) => {
let ei = ei.item;
let tslast = if ei.len() > 0 { Some(ei.ts(ei.len() - 1)) } else { None };
i1 += 1;
if i1 % 1000 == 0 {
info!("read msg from file {}", i1);
}
let ei2 = ei.x_aggregate(&evq.agg_kind);
let g = Ok(StreamItem::DataItem(RangeCompletableItem::Data(ei2)));
tx.send(g).await.errstr()?;
if let Some(t) = tslast {
if t >= evq.range.end {
info!("after requested range, break");
break 'evread;
}
}
}
Ok(None) => {
debug!("reached end of file");
break;
}
Err(e) => {
error!("error while reading msg {:?}", e);
break;
}
}
}
}
}
Err(e) => {
error!("bad filename parse {:?}", e);
}
}
} else {
debug!("prefix {} s {}", prefix, s);
}
}
Ok::<_, Error>(())
};
let block2 = async move {
match block1.await {
Ok(_) => {}
Err(e) => {
error!("{:?}", e);
}
}
};
tokio::task::spawn(block2);
Ok(Box::pin(rx))
}
pub enum PositionState {
NothingFound,
Positioned(u64),
}
pub struct PositionResult {
pub pbr: PbFileReader,
pub state: PositionState,
}
pub async fn position_file_for_evq(mut file: File, evq: RawEventsQuery, year: u32) -> Result<PositionResult, Error> {
trace!("-------------- position_file_for_evq");
let flen = file.seek(SeekFrom::End(0)).await?;
file.seek(SeekFrom::Start(0)).await?;
if true || flen < 1024 * 512 {
position_file_for_evq_linear(file, evq, year).await
} else {
position_file_for_evq_binary(file, evq, year).await
}
}
async fn position_file_for_evq_linear(file: File, evq: RawEventsQuery, _year: u32) -> Result<PositionResult, Error> {
// TODO make read of header part of init:
let mut pbr = PbFileReader::new(file).await?;
let mut curpos;
loop {
// TODO
// Issue is that I always read more than the actual packet.
// Is protobuf length-framed?
// Otherwise: read_header must return the number of bytes that were read.
curpos = pbr.abspos();
trace!("position_file_for_evq_linear save curpos {}", curpos);
let res = pbr.read_msg().await?;
match res {
Some(res) => {
trace!(
"position_file_for_evq_linear read_msg pos {} len {}",
res.pos,
res.item.len()
);
if res.item.len() < 1 {
return Err(Error::with_msg_no_trace("no event read from file"));
}
let tslast = res.item.ts(res.item.len() - 1);
let diff = tslast as i64 - evq.range.beg as i64;
trace!("position_file_for_evq_linear tslast {} diff {}", tslast, diff);
if tslast >= evq.range.beg {
debug!("position_file_for_evq_linear Positioned curpos {}", curpos);
pbr.reset_io(curpos).await?;
let ret = PositionResult {
state: PositionState::Positioned(curpos),
pbr,
};
return Ok(ret);
}
}
None => {
debug!("position_file_for_evq_linear NothingFound");
pbr.reset_io(0).await?;
let ret = PositionResult {
state: PositionState::NothingFound,
pbr,
};
return Ok(ret);
}
}
}
}
async fn position_file_for_evq_binary(mut file: File, evq: RawEventsQuery, year: u32) -> Result<PositionResult, Error> {
debug!("position_file_for_evq_binary");
let flen = file.seek(SeekFrom::End(0)).await?;
file.seek(SeekFrom::Start(0)).await?;
// TODO make read of header part of init:
let mut pbr = PbFileReader::new(file).await?;
let payload_type = pbr.payload_type().clone();
let res = pbr.read_msg().await?;
//let mut file = pbr.into_file();
let file = pbr.file();
let res = if let Some(res) = res {
res
} else {
return Err(Error::with_msg_no_trace("no event read from file"));
};
if res.item.len() < 1 {
return Err(Error::with_msg_no_trace("no event read from file"));
}
let events_begin_pos = res.pos;
// * the search invariant is that the ts1 < beg and ts2 >= end
// * read some data from the end.
// * read some data from the begin.
// * extract events from begin and end.
// * check if the binary search invariant is already violated, in that case return.
// * otherwise, choose some spot in the middle, read there the next chunk.
// Then use the actual position of the found item!
let mut buf1 = vec![0; 1024 * 16];
let mut buf2 = vec![0; 1024 * 16];
let mut buf3 = vec![0; 1024 * 16];
let mut p1 = events_begin_pos;
let mut p2 = flen - buf2.len() as u64;
file.seek(SeekFrom::Start(p1 - 1)).await?;
file.read_exact(&mut buf1).await?;
file.seek(SeekFrom::Start(p2)).await?;
file.read_exact(&mut buf2).await?;
let evs1 = parse_all_ts(p1 - 1, &buf1, payload_type.clone(), year)?;
let evs2 = parse_all_ts(p2, &buf2, payload_type.clone(), year)?;
debug!("...............................................................");
debug!("evs1.len() {:?}", evs1.len());
debug!("evs2.len() {:?}", evs2.len());
debug!("p1: {}", p1);
debug!("p2: {}", p2);
let tgt = evq.range.beg;
{
let ev = evs1.first().unwrap();
if ev.ts >= tgt {
pbr.reset_io(ev.pos).await?;
let ret = PositionResult {
state: PositionState::Positioned(ev.pos),
pbr,
};
return Ok(ret);
}
}
{
let ev = evs2.last().unwrap();
if ev.ts < tgt {
pbr.reset_io(0).await?;
let ret = PositionResult {
state: PositionState::NothingFound,
pbr,
};
return Ok(ret);
}
}
p2 = evs2.last().unwrap().pos;
// TODO make sure that NL-delimited chunks have a max size.
loop {
info!("bsearch loop p1 {} p2 {}", p1, p2);
if p2 - p1 < 1024 * 128 {
// TODO switch here to linear search...
info!("switch to linear search in pos {}..{}", p1, p2);
return linear_search_2(pbr, evq, year, p1, p2, payload_type).await;
}
let p3 = (p2 + p1) / 2;
file.seek(SeekFrom::Start(p3)).await?;
file.read_exact(&mut buf3).await?;
let evs3 = parse_all_ts(p3, &buf3, payload_type.clone(), year)?;
let ev = evs3.first().unwrap();
if ev.ts < tgt {
info!("p3 {} ts: {} pos: {} branch A", p3, ev.ts, ev.pos);
p1 = ev.pos;
} else {
info!("p3 {} ts: {} pos: {} branch B", p3, ev.ts, ev.pos);
p2 = ev.pos;
}
}
}
async fn linear_search_2(
mut pbr: PbFileReader,
evq: RawEventsQuery,
year: u32,
p1: u64,
p2: u64,
payload_type: PayloadType,
) -> Result<PositionResult, Error> {
debug!("linear_search_2 begin");
// TODO improve.. either use additional file handle, or keep pbr in consistent state.
let file = pbr.file();
file.seek(SeekFrom::Start(p1 - 1)).await?;
let mut buf = vec![0; (p2 - p1) as usize];
file.read_exact(&mut buf).await?;
let evs1 = parse_all_ts(p1 - 1, &buf, payload_type.clone(), year)?;
for ev in evs1 {
if ev.ts >= evq.range.beg {
debug!("linear_search_2 Positioned {:?}", ev);
pbr.reset_io(ev.pos).await?;
let ret = PositionResult {
state: PositionState::Positioned(ev.pos),
pbr,
};
return Ok(ret);
}
}
Err(Error::with_msg_no_trace("linear_search_2 failed"))
}
#[allow(unused)]
fn events_item_to_framable(ei: EventsItem) -> Result<Box<dyn Framable + Send>, Error> {
match ei {
EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I32(h))) => {
let range: NanoRange = err::todoval();
let (tss, pulses, values) = h
.tss
.into_iter()
.zip(h.pulses.into_iter())
.zip(h.values.into_iter())
.filter_map(|((t, p), v)| {
if t < range.beg || t >= range.end {
None
} else {
Some((t, p, v))
}
})
.fold((vec![], vec![], vec![]), |(mut a, mut b, mut c), (j, k, l)| {
a.push(j);
b.push(k);
c.push(l);
(a, b, c)
});
let b = ScalarEvents { tss, pulses, values };
let b = Ok(StreamItem::DataItem(RangeCompletableItem::Data(b)));
let ret = Box::new(b);
Ok(ret)
}
_ => {
error!("case not covered");
Err(Error::with_msg_no_trace("todo"))
}
}
}
#[derive(Debug)]
pub struct DirAndPrefix {
dir: PathBuf,
prefix: String,
}
pub fn directory_for_channel_files(channel: &Channel, base_path: &PathBuf) -> Result<DirAndPrefix, ArchError> {
// SARUN11/CVME/DBLM546/IOC_CPU_LOAD
// SARUN11-CVME-DBLM546:IOC_CPU_LOAD
let a: Vec<_> = channel.name.split("-").map(|s| s.split(":")).flatten().collect();
let path = base_path;
let path = a.iter().take(a.len() - 1).fold(path.clone(), |a, &x| a.join(x));
let ret = DirAndPrefix {
dir: path,
prefix: a
.last()
.ok_or_else(|| ArchError::with_msg_no_trace("no prefix in file"))?
.to_string(),
};
Ok(ret)
}
// The same channel-name in different data directories like "lts", "mts", .. are considered different channels.
pub async fn find_files_for_channel(base_path: &PathBuf, channel: &Channel) -> Result<Vec<PathBuf>, ArchError> {
let mut ret = vec![];
let chandir = directory_for_channel_files(channel, base_path)?;
let mut rd = read_dir(&chandir.dir).await?;
while let Some(en) = rd.next_entry().await? {
let fns = en.file_name().to_string_lossy().into_owned();
if fns.starts_with(&format!("{}:20", chandir.prefix)) && fns.ends_with(".pb") {
ret.push(en.path());
}
}
ret.sort_unstable();
Ok(ret)
}
pub async fn channel_info(channel: &Channel, aa: &ArchiverAppliance) -> Result<ChannelInfo, Error> {
let DirAndPrefix { dir, prefix } = directory_for_channel_files(channel, aa.data_base_paths.last().unwrap())?;
let mut msgs = vec![];
msgs.push(format!("path: {}", dir.to_string_lossy()));
let mut scalar_type = None;
let mut shape = None;
let mut rd = read_dir(&dir)
.await
.map_err(|e| Error::with_msg(format!("Can not open directory {dir:?} {e:?}")))?;
while let Some(de) = rd.next_entry().await? {
let s = de.file_name().to_string_lossy().into_owned();
if s.starts_with(&prefix) && s.ends_with(".pb") {
msgs.push(s);
let f1 = File::open(de.path()).await?;
let mut pbr = PbFileReader::new(f1).await?;
msgs.push(format!("got header {}", pbr.channel_name()));
let ev = pbr.read_msg().await;
match ev {
Ok(Some(item)) => {
let item = item.item;
msgs.push(format!("got event {:?}", item));
shape = Some(item.shape());
// These type mappings are defined by the protobuffer schema.
scalar_type = Some(item.scalar_type());
break;
}
Ok(None) => {
msgs.push(format!("can not read event"));
}
Err(e) => {
msgs.push(format!("can not read event {:?}", e));
}
}
msgs.push(format!("got header {}", pbr.channel_name()));
}
}
let shape = shape.ok_or_else(|| Error::with_msg(format!("could not determine shape {:?}", msgs)))?;
let scalar_type =
scalar_type.ok_or_else(|| Error::with_msg(format!("could not determine scalar_type {:?}", msgs)))?;
let ret = ChannelInfo {
scalar_type,
byte_order: None,
shape,
msg: JsonValue::Array(msgs.into_iter().map(JsonValue::String).collect()),
};
Ok(ret)
}
archapp/src/generated.rs
-2
View File
@@ -1,2 +0,0 @@
#[allow(non_snake_case)]
pub mod EPICSEvent;
archapp/src/generated/EPICSEvent.rs
-8684
View File
File diff suppressed because it is too large Load Diff
archapp/src/lib.rs
-76
View File
@@ -1,76 +0,0 @@
#[cfg(feature = "devread")]
pub mod generated;
#[cfg(not(feature = "devread"))]
pub mod generated {}
pub mod archeng;
pub mod err;
pub mod events;
#[cfg(feature = "devread")]
pub mod parse;
#[cfg(not(feature = "devread"))]
pub mod parsestub;
pub mod storagemerge;
#[cfg(feature = "devread")]
#[cfg(test)]
pub mod test;
pub mod timed;
use ::err::Error;
use async_channel::Sender;
use futures_core::Future;
use netpod::log::*;
#[cfg(not(feature = "devread"))]
pub use parsestub as parse;
use std::sync::atomic::{AtomicUsize, Ordering};
fn unescape_archapp_msg(inp: &[u8], mut ret: Vec<u8>) -> Result<Vec<u8>, Error> {
ret.clear();
let mut esc = false;
for &k in inp.iter() {
if k == 0x1b {
esc = true;
} else if esc {
if k == 0x1 {
ret.push(0x1b);
} else if k == 0x2 {
ret.push(0xa);
} else if k == 0x3 {
ret.push(0xd);
} else {
return Err(Error::with_msg_no_trace("malformed escaped archapp message"));
}
esc = false;
} else {
ret.push(k);
}
}
Ok(ret)
}
static CHANNEL_SEND_ERROR: AtomicUsize = AtomicUsize::new(0);
fn channel_send_error() {
let c = CHANNEL_SEND_ERROR.fetch_add(1, Ordering::AcqRel);
if c < 10 {
error!("CHANNEL_SEND_ERROR {}", c);
}
}
fn wrap_task<T, O1, O2>(task: T, tx: Sender<Result<O2, Error>>)
where
T: Future<Output = Result<O1, Error>> + Send + 'static,
O1: Send + 'static,
O2: Send + 'static,
{
let task = async move {
match task.await {
Ok(_) => {}
Err(e) => {
if let Err(_) = tx.send(Err(e)).await {
channel_send_error();
}
}
}
};
taskrun::spawn(task);
}
archapp/src/parse.rs
-617
View File
@@ -1,617 +0,0 @@
pub mod multi;
use crate::events::parse_data_filename;
use crate::generated::EPICSEvent::PayloadType;
use crate::unescape_archapp_msg;
use archapp_xc::*;
use async_channel::{bounded, Receiver};
use chrono::{TimeZone, Utc};
use err::{ErrStr, Error};
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use items::scalarevents::ScalarEvents;
use items::waveevents::WaveEvents;
use netpod::log::*;
use netpod::{ArchiverAppliance, ChannelConfigQuery, ChannelConfigResponse};
use netpod::{Database, ScalarType, Shape};
use protobuf::Message;
use serde::Serialize;
use std::collections::{BTreeMap, VecDeque};
use std::fs::FileType;
use std::io::SeekFrom;
use std::mem;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
pub struct PbFileReader {
file: File,
buf: Vec<u8>,
escbuf: Vec<u8>,
wp: usize,
rp: usize,
off: u64,
channel_name: String,
payload_type: PayloadType,
year: u32,
}
fn parse_scalar_byte(m: &[u8], year: u32) -> Result<EventsItem, Error> {
let msg = crate::generated::EPICSEvent::ScalarByte::parse_from_bytes(m)
.map_err(|_| Error::with_msg(format!("can not parse pb-type {}", "ScalarByte")))?;
let mut t = ScalarEvents::<i8>::empty();
let yd = Utc.ymd(year as i32, 1, 1).and_hms(0, 0, 0);
let ts = yd.timestamp() as u64 * 1000000000 + msg.get_secondsintoyear() as u64 * 1000000000 + msg.get_nano() as u64;
let v = msg.get_val().first().map_or(0, |k| *k as i8);
t.tss.push(ts);
t.values.push(v);
Ok(EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::I8(t))))
}
macro_rules! scalar_parse {
($m:expr, $year:expr, $pbt:ident, $eit:ident, $evty:ident) => {{
let msg = crate::generated::EPICSEvent::$pbt::parse_from_bytes($m)
.map_err(|e| Error::with_msg(format!("can not parse pb-type {} {:?}", stringify!($pbt), e)))?;
let mut t = ScalarEvents::<$evty>::empty();
let yd = Utc.ymd($year as i32, 1, 1).and_hms(0, 0, 0);
let ts =
yd.timestamp() as u64 * 1000000000 + msg.get_secondsintoyear() as u64 * 1000000000 + msg.get_nano() as u64;
let v = msg.get_val();
//eprintln!("ts {} val {}", ts, v);
t.tss.push(ts);
t.values.push(v as $evty);
EventsItem::Plain(PlainEvents::Scalar(ScalarPlainEvents::$eit(t)))
}};
}
macro_rules! wave_parse {
($m:expr, $year:expr, $pbt:ident, $eit:ident, $evty:ident) => {{
let msg = crate::generated::EPICSEvent::$pbt::parse_from_bytes($m)
.map_err(|_| Error::with_msg(format!("can not parse pb-type {}", stringify!($pbt))))?;
let mut t = WaveEvents::<$evty>::empty();
let yd = Utc.ymd($year as i32, 1, 1).and_hms(0, 0, 0);
let ts =
yd.timestamp() as u64 * 1000000000 + msg.get_secondsintoyear() as u64 * 1000000000 + msg.get_nano() as u64;
let v = msg.get_val();
t.tss.push(ts);
t.vals.push(v.into_iter().map(|&x| x as $evty).collect());
EventsItem::Plain(PlainEvents::Wave(WavePlainEvents::$eit(t)))
}};
}
const MIN_BUF_FILL: usize = 1024 * 64;
pub struct ReadMessageResult {
pub pos: u64,
pub item: EventsItem,
}
impl PbFileReader {
pub async fn new(file: File) -> Result<Self, Error> {
let mut ret = Self {
file,
buf: vec![0; MIN_BUF_FILL * 4],
escbuf: vec![],
wp: 0,
rp: 0,
// TODO check usage of `off`.
// It should represent the absolute position where the 1st byte of `buf` is located
// in the file, independent of `wp` or `rp`.
off: 0,
channel_name: String::new(),
payload_type: PayloadType::V4_GENERIC_BYTES,
year: 0,
};
ret.read_header().await?;
Ok(ret)
}
pub fn into_file(self) -> File {
self.file
}
pub fn file(&mut self) -> &mut File {
&mut self.file
}
pub async fn reset_io(&mut self, off: u64) -> Result<(), Error> {
self.file().seek(SeekFrom::Start(off)).await?;
self.wp = 0;
self.rp = 0;
self.off = off;
Ok(())
}
pub async fn read_header(&mut self) -> Result<(), Error> {
self.fill_buf().await?;
let k = self.find_next_nl()?;
trace!("read_header abspos {} packet len {}", self.abspos(), k + 1 - self.rp);
let buf = &mut self.buf;
let m = unescape_archapp_msg(&buf[self.rp..k], mem::replace(&mut self.escbuf, vec![]))?;
self.escbuf = m;
let payload_info = crate::generated::EPICSEvent::PayloadInfo::parse_from_bytes(&self.escbuf)
.map_err(|_| Error::with_msg("can not parse PayloadInfo"))?;
self.channel_name = payload_info.get_pvname().into();
self.payload_type = payload_info.get_field_type();
self.year = payload_info.get_year() as u32;
self.rp = k + 1;
Ok(())
}
pub async fn read_msg(&mut self) -> Result<Option<ReadMessageResult>, Error> {
self.fill_buf().await?;
let k = if let Ok(k) = self.find_next_nl() {
k
} else {
debug!("Can not find a next NL");
return Ok(None);
};
//info!("read_msg abspos {} packet len {}", self.abspos(), k + 1 - self.rp);
let buf = &mut self.buf;
let m = mem::replace(&mut self.escbuf, vec![]);
let m = unescape_archapp_msg(&buf[self.rp..k], m)?;
self.escbuf = m;
let ei = Self::parse_buffer(&self.escbuf, self.payload_type.clone(), self.year)?;
let ret = ReadMessageResult {
pos: self.off + self.rp as u64,
item: ei,
};
self.rp = k + 1;
Ok(Some(ret))
}
pub fn parse_buffer(m: &[u8], payload_type: PayloadType, year: u32) -> Result<EventsItem, Error> {
use PayloadType::*;
let ei = match payload_type {
SCALAR_BYTE => parse_scalar_byte(m, year)?,
SCALAR_ENUM => {
scalar_parse!(m, year, ScalarEnum, I32, i32)
}
SCALAR_SHORT => {
scalar_parse!(m, year, ScalarShort, I16, i16)
}
SCALAR_INT => {
scalar_parse!(m, year, ScalarInt, I32, i32)
}
SCALAR_FLOAT => {
scalar_parse!(m, year, ScalarFloat, F32, f32)
}
SCALAR_DOUBLE => {
scalar_parse!(m, year, ScalarDouble, F64, f64)
}
WAVEFORM_BYTE => {
wave_parse!(m, year, VectorChar, I8, i8)
}
WAVEFORM_SHORT => {
wave_parse!(m, year, VectorShort, I16, i16)
}
WAVEFORM_ENUM => {
wave_parse!(m, year, VectorEnum, I32, i32)
}
WAVEFORM_INT => {
wave_parse!(m, year, VectorInt, I32, i32)
}
WAVEFORM_FLOAT => {
wave_parse!(m, year, VectorFloat, F32, f32)
}
WAVEFORM_DOUBLE => {
wave_parse!(m, year, VectorDouble, F64, f64)
}
SCALAR_STRING | WAVEFORM_STRING | V4_GENERIC_BYTES => {
return Err(Error::with_msg_no_trace(format!("not supported: {:?}", payload_type)));
}
};
Ok(ei)
}
pub fn abspos(&self) -> u64 {
self.off + self.rp as u64
}
async fn fill_buf(&mut self) -> Result<(), Error> {
if self.wp - self.rp >= MIN_BUF_FILL {
return Ok(());
}
if self.rp + MIN_BUF_FILL >= self.buf.len() {
let n = self.wp - self.rp;
self.buf.copy_within(self.rp..self.rp + n, 0);
self.off += self.rp as u64;
self.rp = 0;
self.wp = n;
}
let buf = &mut self.buf;
loop {
let sl = &mut buf[self.wp..];
if sl.len() == 0 {
break;
}
let n = self.file.read(sl).await?;
if n == 0 {
break;
} else {
self.wp += n;
}
}
Ok(())
}
fn find_next_nl(&self) -> Result<usize, Error> {
let buf = &self.buf;
let mut k = self.rp;
while k < self.wp && buf[k] != 0xa {
k += 1;
}
if k == self.wp {
// TODO test whether with_msg_no_trace makes difference.
return Err(Error::with_msg("no nl in pb file"));
}
Ok(k)
}
pub fn channel_name(&self) -> &str {
&self.channel_name
}
pub fn payload_type(&self) -> &PayloadType {
&self.payload_type
}
}
#[derive(Serialize)]
pub struct EpicsEventPayloadInfo {
headers: Vec<(String, String)>,
year: i32,
pvname: String,
datatype: String,
ts0: u32,
val0: f32,
}
// TODO remove in favor of PbFileRead
async fn _read_pb_file(mut f1: File) -> Result<(EpicsEventPayloadInfo, File), Error> {
let mut buf = vec![0; 1024 * 4];
{
let mut i1 = 0;
loop {
let n = f1.read(&mut buf[i1..]).await?;
if n == 0 {
break;
}
i1 += n;
if i1 >= buf.len() {
break;
}
}
}
let mut j1 = 0;
let mut payload_info = crate::generated::EPICSEvent::PayloadInfo::new();
let mut z = EpicsEventPayloadInfo {
pvname: String::new(),
headers: vec![],
year: 0,
datatype: String::new(),
ts0: 0,
val0: 0.0,
};
loop {
let mut i2 = usize::MAX;
for (i1, &k) in buf[j1..].iter().enumerate() {
if k == 0xa {
i2 = j1 + i1;
break;
}
}
if i2 != usize::MAX {
//info!("got NL {} .. {}", j1, i2);
let m = unescape_archapp_msg(&buf[j1..i2], vec![])?;
if j1 == 0 {
payload_info = crate::generated::EPICSEvent::PayloadInfo::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse PayloadInfo"))?;
//info!("got payload_info: {:?}", payload_info);
z = EpicsEventPayloadInfo {
headers: payload_info
.get_headers()
.iter()
.map(|j| (j.get_name().to_string(), j.get_val().to_string()))
.collect(),
year: payload_info.get_year(),
pvname: payload_info.get_pvname().into(),
datatype: String::new(),
ts0: 0,
val0: 0.0,
};
} else {
let ft = payload_info.get_field_type();
{
use crate::generated::EPICSEvent::PayloadType::*;
let (ts, val) = match ft {
SCALAR_BYTE => {
let d = crate::generated::EPICSEvent::ScalarByte::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::ScalarByte"))?;
(d.get_secondsintoyear(), d.get_val()[0] as f32)
}
SCALAR_SHORT => {
let d = crate::generated::EPICSEvent::ScalarShort::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::ScalarShort"))?;
(d.get_secondsintoyear(), d.get_val() as f32)
}
SCALAR_INT => {
let d = crate::generated::EPICSEvent::ScalarInt::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::ScalarInt"))?;
(d.get_secondsintoyear(), d.get_val() as f32)
}
SCALAR_FLOAT => {
let d = crate::generated::EPICSEvent::ScalarFloat::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::ScalarFloat"))?;
(d.get_secondsintoyear(), d.get_val() as f32)
}
SCALAR_DOUBLE => {
let d = crate::generated::EPICSEvent::ScalarDouble::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::ScalarDouble"))?;
(d.get_secondsintoyear(), d.get_val() as f32)
}
WAVEFORM_FLOAT => {
let d = crate::generated::EPICSEvent::VectorFloat::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::VectorFloat"))?;
(d.get_secondsintoyear(), d.get_val()[0] as f32)
}
WAVEFORM_DOUBLE => {
let d = crate::generated::EPICSEvent::VectorDouble::parse_from_bytes(&m)
.map_err(|_| Error::with_msg("can not parse EPICSEvent::VectorDouble"))?;
(d.get_secondsintoyear(), d.get_val()[0] as f32)
}
_ => (0, 0.0),
};
z.datatype = format!("{:?}", ft);
z.ts0 = ts;
z.val0 = val;
return Ok((z, f1));
}
}
} else {
//info!("no more packets");
break;
}
j1 = i2 + 1;
}
Err(Error::with_msg(format!("no data found in file")))
}
struct LruCache {
map: BTreeMap<String, Instant>,
}
impl LruCache {
fn new() -> Self {
Self { map: BTreeMap::new() }
}
fn insert(&mut self, key: &str) {
self.map.insert(key.into(), Instant::now());
if self.map.len() > 2000 {
let mut tss: Vec<Instant> = self.map.values().map(|j| j.clone()).collect();
tss.sort_unstable();
let thr = tss[1500];
let m1 = std::mem::replace(&mut self.map, BTreeMap::new());
self.map = m1.into_iter().filter(|(_j, k)| k > &thr).collect();
}
}
fn query(&self, key: &str) -> bool {
self.map.get(key).map_or(false, |_| true)
}
}
#[derive(Debug)]
#[allow(unused)]
pub struct DiscoveredDatafile {
path: PathBuf,
channel_name: String,
variant_name: String,
scalar_type: ScalarType,
shape: Shape,
}
pub async fn scan_files_inner(
pairs: BTreeMap<String, String>,
data_base_paths: Vec<PathBuf>,
) -> Result<Receiver<Result<DiscoveredDatafile, Error>>, Error> {
let _ = pairs;
let (tx, rx) = bounded(16);
let tx = Arc::new(tx);
let tx2 = tx.clone();
let block1 = async move {
let mut lru = LruCache::new();
struct PE {
path: PathBuf,
fty: FileType,
}
let proot = data_base_paths.last().unwrap().clone();
let proots = proot.to_str().unwrap().to_string();
// TODO factor out to automatically add path to error.
let meta = tokio::fs::metadata(&proot)
.await
.map_err(|e| Error::with_msg(format!("can not open {proot:?} {e:?}")))?;
let mut paths = VecDeque::new();
paths.push_back(PE {
path: proot,
fty: meta.file_type(),
});
loop {
if let Some(pe) = paths.pop_back() {
if pe.fty.is_dir() {
// TODO factor out to automatically add path to error.
let mut rd = tokio::fs::read_dir(&pe.path)
.await
.map_err(|e| Error::with_msg(format!("can not open {:?} {e:?}", pe.path)))?;
loop {
match rd.next_entry().await {
Ok(item) => match item {
Some(item) => {
paths.push_back(PE {
path: item.path(),
fty: item.file_type().await?,
});
}
None => {
break;
}
},
Err(e) => {
tx.send(Err(e.into())).await.errstr()?;
}
}
}
} else if pe.fty.is_file() {
//tx.send(Ok(Box::new(path.clone()) as RT1)).await?;
let fns = pe.path.to_str().ok_or_else(|| Error::with_msg("invalid path string"))?;
if let Ok(_fnp) = parse_data_filename(&fns) {
//tx.send(Ok(Box::new(serde_json::to_value(fns)?) as ItemSerBox)).await?;
let channel_path = &fns[proots.len() + 1..fns.len() - 11];
if !lru.query(channel_path) {
let f3 = tokio::fs::File::open(&pe.path)
.await
.map_err(|e| Error::with_msg(format!("can not open {:?} {e:?}", pe.path)))?;
let mut pbr = PbFileReader::new(f3).await?;
let normalized_channel_name = {
let pvn = pbr.channel_name().replace("-", "/");
pvn.replace(":", "/")
};
if channel_path != normalized_channel_name {
let msg = format!("{} - {}", channel_path, normalized_channel_name);
warn!("{}", msg);
tx.send(Err(Error::with_msg(format!("MISMATCH --------------------"))))
.await
.errstr()?;
} else {
if let Ok(Some(msg)) = pbr.read_msg().await {
lru.insert(channel_path);
let item = DiscoveredDatafile {
path: pe.path,
channel_name: pbr.channel_name().into(),
variant_name: msg.item.variant_name(),
scalar_type: msg.item.type_info().0,
shape: msg.item.type_info().1,
};
tx.send(Ok(item)).await.errstr()?;
}
}
}
}
}
} else {
break;
}
}
Ok::<_, Error>(())
};
let block2 = async move {
match block1.await {
Ok(_) => {}
Err(e) => match tx2.send(Err(e)).await {
Ok(_) => {}
Err(e) => {
error!("can not deliver error through channel: {:?}", e);
}
},
}
};
tokio::spawn(block2);
Ok(rx)
}
pub async fn scan_files_msgs(
pairs: BTreeMap<String, String>,
data_base_paths: Vec<PathBuf>,
) -> Result<Receiver<Result<ItemSerBox, Error>>, Error> {
let (tx, rx) = bounded(16);
let tx = Arc::new(tx);
let tx2 = tx.clone();
let block1 = async move {
let mut inp = scan_files_inner(pairs, data_base_paths).await?;
while let Some(item) = inp.next().await {
let item = item?;
let msg = format!("{item:?}");
tx.send(Ok(Box::new(serde_json::to_value(msg)?) as ItemSerBox))
.await
.errstr()?;
}
Ok(())
};
let block2 = async move {
match block1.await {
Ok(_) => {}
Err(e) => match tx2.send(Err(e)).await {
Ok(_) => {}
Err(e) => {
error!("can not deliver error through channel: {:?}", e);
}
},
}
};
tokio::spawn(block2);
Ok(rx)
}
pub async fn scan_files_to_database(
pairs: BTreeMap<String, String>,
data_base_paths: Vec<PathBuf>,
database: Database,
) -> Result<Receiver<Result<ItemSerBox, Error>>, Error> {
let (tx, rx) = bounded(16);
let tx = Arc::new(tx);
let tx2 = tx.clone();
let block1 = async move {
let dbc = dbconn::create_connection(&database).await?;
let mut inp = scan_files_inner(pairs, data_base_paths).await?;
while let Some(item) = inp.next().await {
let item = item?;
let sql = "select rowid from channels where name = $1";
let rows = dbc.query(sql, &[&item.channel_name]).await.errstr()?;
if rows.len() == 0 {
let sql = "insert into channels (name, config) values ($1, $2) on conflict do nothing returning rowid";
let cfg = serde_json::json!({
"scalarType":item.scalar_type.to_bsread_str(),
"shape":item.shape,
});
let rows = dbc.query(sql, &[&item.channel_name, &cfg]).await.errstr()?;
if let Some(row) = rows.last() {
let rowid: i64 = row.get(0);
let msg = format!("insert done rowid {rowid} {item:?}");
let msg = Box::new(serde_json::to_value(msg)?) as ItemSerBox;
tx.send(Ok(msg)).await.errstr()?;
}
} else {
// TODO update channel config if needed
}
}
Ok(())
};
let block2 = async move {
match block1.await {
Ok(_) => {}
Err(e) => match tx2.send(Err(e)).await {
Ok(_) => {}
Err(e) => {
error!("can not deliver error through channel: {:?}", e);
}
},
}
};
tokio::spawn(block2);
Ok(rx)
}
pub async fn channel_config(q: &ChannelConfigQuery, aa: &ArchiverAppliance) -> Result<ChannelConfigResponse, Error> {
let ci = crate::events::channel_info(&q.channel, aa).await?;
let ret = ChannelConfigResponse {
channel: q.channel.clone(),
scalar_type: ci.scalar_type,
byte_order: ci.byte_order,
shape: ci.shape,
};
Ok(ret)
}
archapp/src/parse/multi.rs
-53
View File
@@ -1,53 +0,0 @@
use crate::generated::EPICSEvent::PayloadType;
use crate::parse::PbFileReader;
use err::Error;
use items::{WithLen, WithTimestamps};
use netpod::log::*;
#[derive(Debug)]
pub struct PosTs {
pub pos: u64,
pub ts: u64,
}
pub fn parse_all_ts(off: u64, buf: &[u8], payload_type: PayloadType, year: u32) -> Result<Vec<PosTs>, Error> {
let mut ret = vec![];
let mut i1 = 0;
let mut i2 = usize::MAX;
loop {
if i1 >= buf.len() {
break;
}
if buf[i1] == 10 {
if i2 == usize::MAX {
i2 = i1;
} else {
// Have a chunk from i2..i1
info!("call parse_buffer i2 {} i1 {}", i2, i1);
match PbFileReader::parse_buffer(&buf[i2 + 1..i1], payload_type.clone(), year) {
Ok(k) => {
if k.len() != 1 {
return Err(Error::with_msg_no_trace(format!(
"parsed buffer contained {} events",
k.len()
)));
} else {
let h = PosTs {
pos: off + i2 as u64 + 1,
ts: k.ts(0),
};
ret.push(h);
}
}
Err(e) => {
error!("parse_all_ts: {:?}", e);
return Err(e);
}
}
i2 = i1;
}
}
i1 += 1;
}
Ok(ret)
}
archapp/src/parsestub.rs
-14
View File
@@ -1,14 +0,0 @@
use archapp_xc::ItemSer;
use async_channel::Receiver;
use err::Error;
use netpod::NodeConfigCached;
use std::collections::BTreeMap;
type RT1 = Box<dyn ItemSer + Send>;
pub async fn scan_files(
_pairs: BTreeMap<String, String>,
_node_config: NodeConfigCached,
) -> Result<Receiver<Result<RT1, Error>>, Error> {
Err(Error::with_msg("feature not enabled"))
}
archapp/src/storagemerge.rs
-309
View File
@@ -1,309 +0,0 @@
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use items::eventsitem::EventsItem;
use items::{
inspect_timestamps, Appendable, LogItem, PushableIndex, RangeCompletableItem, Sitemty, StatsItem, StreamItem,
};
use netpod::log::*;
use netpod::{NanoRange, Nanos};
use std::collections::VecDeque;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::{SystemTime, UNIX_EPOCH};
/**
Priority-Merge events from different candidate sources.
Backends like Channel Archiver store the compacted "medium/long-term" data of a channel
in logically unrelated locations on disk with unspecified semantics and without a
common index over "short+medium+long term" data.
In order to deliver data even over the edge of such (possibly overlapping) datasources
without common look tables, the best we can do is fetch data from all sources and
combine them. StorageMerge is doing this combination.
*/
pub struct StorageMerge {
inps: Vec<Pin<Box<dyn Stream<Item = Sitemty<EventsItem>> + Send>>>,
names: Vec<String>,
range: NanoRange,
completed_inps: Vec<bool>,
range_complete: Vec<bool>,
current_inp_item: Vec<Option<EventsItem>>,
error_items: VecDeque<Error>,
log_items: VecDeque<LogItem>,
stats_items: VecDeque<StatsItem>,
ourname: String,
inprng: usize,
data_done: bool,
done: bool,
complete: bool,
}
impl StorageMerge {
pub fn new(
inps: Vec<Pin<Box<dyn Stream<Item = Sitemty<EventsItem>> + Send>>>,
names: Vec<String>,
range: NanoRange,
) -> Self {
assert_eq!(inps.len(), names.len());
let n = inps.len();
let mut h = crc32fast::Hasher::new();
h.update(
&SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.subsec_nanos()
.to_le_bytes(),
);
let ourname = format!("{:08x}", h.finalize());
for (i, n) in names.iter().enumerate() {
debug!("[{}] {} {}", ourname, i, n);
}
Self {
inps,
names,
range,
completed_inps: vec![false; n],
range_complete: vec![false; n],
current_inp_item: (0..n).into_iter().map(|_| None).collect(),
error_items: VecDeque::new(),
log_items: VecDeque::new(),
stats_items: VecDeque::new(),
inprng: n - 1,
data_done: false,
done: false,
complete: false,
ourname,
}
}
fn refill_if_needed(self: &mut Pin<&mut Self>, cx: &mut Context) -> Result<bool, Error> {
use Poll::*;
let mut is_pending = false;
for i in 0..self.inps.len() {
while self.current_inp_item[i].is_none() && self.completed_inps[i] == false {
match self.inps[i].poll_next_unpin(cx) {
Ready(j) => match j {
Some(j) => match j {
Ok(j) => match j {
StreamItem::DataItem(j) => match j {
RangeCompletableItem::Data(j) => {
self.current_inp_item[i] = Some(j);
}
RangeCompletableItem::RangeComplete => {
self.range_complete[i] = true;
}
},
StreamItem::Log(k) => {
self.log_items.push_back(k);
}
StreamItem::Stats(k) => {
self.stats_items.push_back(k);
}
},
Err(e) => {
error!("inp err input {} {:?}", i, e);
self.error_items.push_back(e);
}
},
None => {
self.completed_inps[i] = true;
}
},
Pending => {
is_pending = true;
break;
}
}
}
}
Ok(is_pending)
}
fn decide_next_item(&mut self) -> Result<Option<Sitemty<EventsItem>>, Error> {
let not_found = 99999;
let mut i1 = self.inprng;
let mut j1 = not_found;
let mut tsmin = u64::MAX;
let mut tsend = u64::MAX;
#[allow(unused)]
use items::{WithLen, WithTimestamps};
loop {
if self.completed_inps[i1] {
} else {
match self.current_inp_item[i1].as_ref() {
None => panic!(),
Some(j) => {
if j.len() == 0 {
j1 = i1;
break;
} else {
let ts1 = j.ts(0);
let ts2 = j.ts(j.len() - 1);
if ts1 == u64::MAX || ts2 == u64::MAX {
panic!();
}
trace!("[{}] consider {} {:?}", self.ourname, i1, Nanos::from_ns(ts1));
if ts1 <= tsmin {
tsmin = ts1;
tsend = ts2;
j1 = i1;
trace!(
"[{}] switch to source {} / {} {}",
self.ourname,
i1,
self.inps.len(),
self.names[i1]
);
self.inprng = i1;
} else {
}
}
}
}
}
if i1 == 0 {
break;
}
i1 -= 1;
}
let i1 = ();
let _ = i1;
if j1 >= not_found {
Ok(None)
} else {
trace!("[{}] decide for source {}", self.ourname, j1);
trace!("[{}] decided tsmin {:?}", self.ourname, Nanos::from_ns(tsmin));
trace!("[{}] decided tsend {:?}", self.ourname, Nanos::from_ns(tsend));
let mut j5 = not_found;
let mut tsmin2 = u64::MAX;
if self.inprng > 0 {
trace!("[{}] locate the next earliest timestamp", self.ourname);
let mut i5 = self.inprng - 1;
loop {
if self.completed_inps[i5] {
} else {
let j = self.current_inp_item[i5].as_ref().unwrap();
if j.len() != 0 {
let ts1 = j.ts(0);
if ts1 == u64::MAX {
panic!();
}
trace!(
"[{}] consider {} {:?} for next earliest",
self.ourname,
i5,
Nanos::from_ns(ts1)
);
if ts1 <= tsmin2 {
tsmin2 = ts1;
j5 = i5;
}
}
}
if i5 == 0 {
break;
}
i5 -= 1;
}
}
trace!(
"[{}] decided tsmin2 {:?} next earliest timestamp source {}",
self.ourname,
Nanos::from_ns(tsmin2),
j5
);
let item = self.current_inp_item[j1].take().unwrap();
let item = if j5 != not_found && tsmin2 != u64::MAX {
if tsend >= tsmin2 {
{
let tsmin = Nanos::from_ns(tsmin);
let tsend = Nanos::from_ns(tsend);
let tsmin2 = Nanos::from_ns(tsmin2);
trace!(
"[{}] NEED TO TRUNCATE THE BLOCK tsmin {:?} tsend {:?} tsmin2 {:?}",
self.ourname,
tsmin,
tsend,
tsmin2
);
}
let mut out = item.empty_like_self();
for i in 0..item.len() {
let ts = item.ts(i);
if ts < tsmin2 {
out.push_index(&item, i);
}
}
out
} else {
item
}
} else {
item
};
trace!("[{}] emit {} events", self.ourname, item.len());
if false {
let s = inspect_timestamps(&item, self.range.clone());
trace!("[{}] timestamps:\n{}", self.ourname, s);
}
Ok(Some(Ok(StreamItem::DataItem(RangeCompletableItem::Data(item)))))
}
}
}
impl Stream for StorageMerge {
type Item = Sitemty<EventsItem>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
use Poll::*;
loop {
break if self.complete {
panic!()
} else if self.done {
self.complete = true;
Ready(None)
} else if let Some(k) = self.error_items.pop_front() {
Ready(Some(Err(k)))
} else if let Some(k) = self.log_items.pop_front() {
Ready(Some(Ok(StreamItem::Log(k))))
} else if let Some(k) = self.stats_items.pop_front() {
Ready(Some(Ok(StreamItem::Stats(k))))
} else if self.data_done {
self.done = true;
continue;
} else {
match self.refill_if_needed(cx) {
Ok(is_pending) => {
if is_pending {
if self.log_items.len() == 0 && self.stats_items.len() == 0 {
Pending
} else {
continue;
}
} else if self.error_items.len() != 0 {
continue;
} else {
match self.decide_next_item() {
Ok(Some(j)) => Ready(Some(j)),
Ok(None) => {
self.data_done = true;
continue;
}
Err(e) => {
error!("impl Stream for StorageMerge {:?}", e);
Ready(Some(Err(e)))
}
}
}
}
Err(e) => {
error!("{}", e);
self.done = true;
Ready(Some(Err(e)))
}
}
};
}
}
}
archapp/src/test.rs
-56
View File
@@ -1,56 +0,0 @@
use crate::unescape_archapp_msg;
use err::Error;
use netpod::log::*;
use protobuf::Message;
pub fn read_pb_dummy() -> Result<(), Error> {
Ok(())
}
#[cfg(feature = "devread")]
#[test]
fn read_pb_00() -> Result<(), Error> {
use std::path::PathBuf;
let block1 = async move {
let homedir = std::env::var("HOME").unwrap();
let path = PathBuf::from(homedir)
.join("daqbuffer-testdata")
.join("archappdata")
.join("lts")
.join("ArchiverStore")
.join("SARUN16")
.join("MQUA080")
.join("X:2021_01.pb");
let f1 = tokio::fs::read(path).await?;
let mut j1 = 0;
// TODO assert more
loop {
let mut i2 = usize::MAX;
for (i1, &k) in f1[j1..].iter().enumerate() {
if k == 0xa {
i2 = j1 + i1;
break;
}
}
if i2 != usize::MAX {
debug!("got NL {} .. {}", j1, i2);
let m = unescape_archapp_msg(&f1[j1..i2], vec![])?;
if j1 == 0 {
let payload_info = crate::generated::EPICSEvent::PayloadInfo::parse_from_bytes(&m).unwrap();
debug!("got payload_info: {:?}", payload_info);
} else {
let scalar_double = crate::generated::EPICSEvent::ScalarDouble::parse_from_bytes(&m).unwrap();
debug!("got scalar_double: {:?}", scalar_double);
}
} else {
debug!("no more packets");
break;
}
j1 = i2 + 1;
}
Ok::<_, Error>(())
};
taskrun::run(block1)?;
Ok(())
}
archapp/src/timed.rs
-27
View File
@@ -1,27 +0,0 @@
use netpod::log::*;
use std::time::Instant;
pub struct Timed {
name: String,
ts1: Instant,
}
impl Timed {
pub fn new<T>(name: T) -> Self
where
T: ToString,
{
Self {
name: name.to_string(),
ts1: Instant::now(),
}
}
}
impl Drop for Timed {
fn drop(&mut self) {
let ts2 = Instant::now();
let dt = ts2.duration_since(self.ts1);
debug!("Timed {} {:?}", self.name, dt);
}
}
archapp_wrap/Cargo.toml
-18
View File
@@ -1,18 +0,0 @@
[package]
name = "archapp_wrap"
version = "0.0.1-a.dev.4"
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2018"
[dependencies]
serde = "1.0.126"
serde_json = "1.0.64"
async-channel = "1.6"
futures-core = "0.3.15"
futures-util = "0.3.15"
err = { path = "../err" }
netpod = { path = "../netpod" }
archapp = { path = "../archapp" }
archapp_xc = { path = "../archapp_xc" }
disk = { path = "../disk" }
items = { path = "../items" }
archapp_wrap/src/lib.rs
-49
View File
@@ -1,49 +0,0 @@
use archapp_xc::ItemSerBox;
use async_channel::Receiver;
use err::Error;
use futures_core::Stream;
use items::Framable;
use netpod::query::RawEventsQuery;
use netpod::{ArchiverAppliance, Channel, ChannelConfigQuery, ChannelConfigResponse, ChannelInfo, NodeConfigCached};
use std::collections::BTreeMap;
use std::future::Future;
use std::pin::Pin;
pub use archapp;
pub fn scan_files(
pairs: BTreeMap<String, String>,
node_config: NodeConfigCached,
) -> Pin<Box<dyn Future<Output = Result<Receiver<Result<ItemSerBox, Error>>, Error>> + Send>> {
Box::pin(archapp::parse::scan_files_msgs(
pairs,
node_config.node.archiver_appliance.unwrap().data_base_paths,
))
}
pub fn scan_files_insert(
pairs: BTreeMap<String, String>,
node_config: NodeConfigCached,
) -> Pin<Box<dyn Future<Output = Result<Receiver<Result<ItemSerBox, Error>>, Error>> + Send>> {
let aa = node_config.node.archiver_appliance.as_ref().unwrap();
Box::pin(archapp::parse::scan_files_to_database(
pairs,
aa.data_base_paths.clone(),
node_config.node_config.cluster.database.clone(),
))
}
pub async fn make_event_pipe(
evq: &RawEventsQuery,
aa: &ArchiverAppliance,
) -> Result<Pin<Box<dyn Stream<Item = Box<dyn Framable + Send>> + Send>>, Error> {
archapp::events::make_event_pipe(evq, aa).await
}
pub async fn channel_info(channel: &Channel, node_config: &NodeConfigCached) -> Result<ChannelInfo, Error> {
archapp::events::channel_info(channel, node_config.node.archiver_appliance.as_ref().unwrap()).await
}
pub async fn channel_config(q: &ChannelConfigQuery, aa: &ArchiverAppliance) -> Result<ChannelConfigResponse, Error> {
archapp::parse::channel_config(q, aa).await
}
archapp_xc/Cargo.toml
-12
View File
@@ -1,12 +0,0 @@
[package]
name = "archapp_xc"
version = "0.0.1-a.dev.4"
authors = ["Dominik Werder <dominik.werder@gmail.com>"]
edition = "2018"
[dependencies]
serde = "1.0.126"
serde_json = "1.0.64"
async-channel = "1.6"
err = { path = "../err" }
netpod = { path = "../netpod" }
archapp_xc/src/lib.rs
-18
View File
@@ -1,18 +0,0 @@
use err::Error;
use serde::Serialize;
pub type ItemSerBox = Box<dyn ItemSer + Send>;
pub trait ItemSer {
fn serialize(&self) -> Result<Vec<u8>, Error>;
}
impl<T> ItemSer for T
where
T: Serialize,
{
fn serialize(&self) -> Result<Vec<u8>, Error> {
let u = serde_json::to_vec(self)?;
Ok(u)
}
}
dq/Cargo.toml
-1
View File
@@ -21,4 +21,3 @@ netpod = { path = "../netpod" }
items = { path = "../items" }
parse = { path = "../parse" }
disk = { path = "../disk" }
archapp = { path = "../archapp" }
dq/src/bin/daq-convert.rs
+5 -10
View File
@@ -1,6 +1,6 @@
use clap::Parser;
use err::Error;
use netpod::{timeunits::*, Nanos};
use netpod::timeunits::*;
use std::{path::PathBuf, str::FromStr};
#[derive(Debug, Parser)]
@@ -39,6 +39,7 @@ pub struct ConvertArchiverApplianceChannel {
#[derive(Clone, Debug)]
pub struct TimeBinSize {
#[allow(unused)]
nanos: u64,
}
@@ -68,15 +69,9 @@ pub fn main() -> Result<(), Error> {
taskrun::run(async {
let opts = Opts::parse();
match opts.subcmd {
SubCmd::ConvertArchiverApplianceChannel(sub) => {
let params = dq::ConvertParams {
keyspace_prefix: sub.keyspace_prefix,
channel_name: sub.channel_name,
input_dir: sub.input_dir,
output_dir: sub.output_dir,
time_bin_size: Nanos::from_ns(sub.time_bin_size.nanos),
};
dq::convert(params).await
SubCmd::ConvertArchiverApplianceChannel(_) => {
eprintln!("error: archapp not built");
Ok(())
}
}
})
dq/src/dq.rs
-455
View File
@@ -1,20 +1,5 @@
use archapp::events::PositionState;
use archapp::parse::PbFileReader;
use bytes::BufMut;
use chrono::{TimeZone, Utc};
use err::Error;
use items::plainevents::{PlainEvents, ScalarPlainEvents, WavePlainEvents};
use netpod::log::*;
use netpod::query::RawEventsQuery;
use netpod::timeunits::*;
use netpod::{AggKind, Channel, HasScalarType, HasShape, NanoRange, Nanos, ScalarType, Shape};
use parse::channelconfig::Config;
use std::fmt;
use std::io::SeekFrom;
use std::mem::take;
use std::path::PathBuf;
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
trait WritableValue: fmt::Debug {
fn put_value(&self, buf: &mut Vec<u8>);
@@ -55,443 +40,3 @@ impl WritableValue for f64 {
buf.put_f64_le(*self);
}
}
struct DataWriter {
output_dir: PathBuf,
kspre: String,
channel: Channel,
bs: Nanos,
tb: u64,
datafile: Option<File>,
indexfile: Option<File>,
wpos: u64,
buf1: Vec<u8>,
}
impl DataWriter {
async fn new(output_dir: PathBuf, kspre: String, channel: Channel, bs: Nanos) -> Result<Self, Error> {
let ret = Self {
output_dir,
kspre,
channel,
bs,
tb: u64::MAX,
datafile: None,
indexfile: None,
wpos: 0,
buf1: vec![0; 1024 * 1024],
};
Ok(ret)
}
async fn write_item(&mut self, item: &PlainEvents) -> Result<(), Error> {
match item {
PlainEvents::Scalar(item) => match item {
ScalarPlainEvents::U32(events) => {
self.write_events(2, ScalarType::U32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I8(events) => {
self.write_events(2, ScalarType::I8, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I16(events) => {
self.write_events(2, ScalarType::I16, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::I32(events) => {
self.write_events(2, ScalarType::I32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::F32(events) => {
self.write_events(2, ScalarType::F32, &events.tss, &events.values)
.await?;
}
ScalarPlainEvents::F64(events) => {
self.write_events(2, ScalarType::F64, &events.tss, &events.values)
.await?;
}
_ => todo!(),
},
PlainEvents::Wave(item) => match item {
WavePlainEvents::F64(_events) => {
todo!()
}
_ => todo!(),
},
}
Ok(())
}
async fn write_events<T: WritableValue>(
&mut self,
ks: u32,
scalar_type: ScalarType,
tss: &Vec<u64>,
vals: &Vec<T>,
) -> Result<(), Error> {
let split = 0;
assert_eq!(tss.len(), vals.len());
for i in 0..tss.len() {
let ts = tss[i];
let tb = ts / self.bs.ns;
if tb != self.tb {
let tbdate = chrono::Utc.timestamp((tb * (self.bs.ns / SEC)) as i64, 0);
eprintln!("Create directory for timebin {}", tbdate);
let p1 = self.output_dir.join(format!("{}_{}", self.kspre, ks));
let p2 = p1.join(self.channel.name());
let p3 = p2.join(format!("{:019}", tb));
let p4 = p3.join(format!("{:010}", split));
let p5 = p4.join(format!("{:019}_00000_Data", self.bs.ns / MS));
let p6 = p4.join(format!("{:019}_00000_Data_Index", self.bs.ns / MS));
tokio::fs::create_dir_all(&p4).await.map_err(|e| {
error!("Can not create {:?}", p4);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p5)
.await
.map_err(|e| {
error!("can not create new file {:?}", p5);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
file.write_all(&len1.to_be_bytes()).await?;
file.write_all(chs).await?;
file.write_all(&len1.to_be_bytes()).await?;
self.wpos = 10 + chs.len() as u64;
self.datafile = Some(file);
if ks == 3 {
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&p6)
.await
.map_err(|e| {
error!("can not create new file {:?}", p6);
e
})?;
file.write_all(&0u16.to_be_bytes()).await?;
self.indexfile = Some(file);
}
self.tb = tb;
}
let file = self.datafile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_i32(0);
buf.put_u64(0);
buf.put_u64(ts);
buf.put_u64(0);
buf.put_u64(0);
// Status, Severity
buf.put_u8(0);
buf.put_u8(0);
buf.put_i32(-1);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(scalar_type.index());
vals[i].put_value(&mut buf);
buf.put_i32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
self.buf1 = buf;
if ks == 3 {
let file = self.indexfile.as_mut().unwrap();
let mut buf = take(&mut self.buf1);
buf.clear();
buf.put_u64(ts);
buf.put_u64(self.wpos);
file.write_all(&buf).await?;
self.buf1 = buf;
}
self.wpos += len1 as u64;
}
Ok(())
}
async fn write_config(&mut self, config: &Config) -> Result<(), Error> {
eprintln!("Create directory for channel config");
let p1 = self.output_dir.join("config").join(self.channel.name()).join("latest");
tokio::fs::create_dir_all(&p1).await.map_err(|e| {
error!("Can not create {:?}", p1);
e
})?;
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(p1.join("00000_Config"))
.await
.map_err(|e| {
error!("can not create config file in {:?}", p1);
e
})?;
let mut buf = take(&mut self.buf1);
{
buf.clear();
buf.put_u16(0);
file.write_all(&buf).await?;
}
{
buf.clear();
let chs = self.channel.name().as_bytes();
let len1 = (chs.len() + 8) as u32;
buf.put_u32(len1);
buf.put_slice(chs);
buf.put_u32(len1);
//let len1 = buf.len();
//buf[0..4].as_mut().put_u32(len1 as u32);
//buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
{
let e = &config.entries[0];
buf.clear();
buf.put_u32(0);
buf.put_u64(0);
buf.put_u64(0);
buf.put_i32(e.ks);
buf.put_u64(e.bs.ns / MS);
buf.put_i32(e.split_count);
buf.put_i32(e.status);
buf.put_i8(e.bb);
buf.put_i32(e.modulo);
buf.put_i32(e.offset);
buf.put_i16(e.precision);
let dtlen = 0;
buf.put_i32(dtlen);
let flags = 0;
buf.put_u8(flags);
buf.put_u8(e.scalar_type.index());
if false {
// is shaped?
buf.put_u8(1);
buf.put_u32(16);
}
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_i32(-1);
buf.put_u32(0);
let len1 = buf.len();
buf[0..4].as_mut().put_u32(len1 as u32);
buf[len1 - 4..len1].as_mut().put_u32(len1 as u32);
file.write_all(&buf).await?;
}
self.buf1 = buf;
Ok(())
}
}
impl Drop for DataWriter {
fn drop(&mut self) {
let indexfile = self.indexfile.take();
let datafile = self.datafile.take();
tokio::task::spawn(async move {
match indexfile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
match datafile {
Some(mut file) => {
let _ = file.flush().await;
}
None => {}
}
});
}
}
#[derive(Clone, Debug)]
pub struct ConvertParams {
pub keyspace_prefix: String,
pub channel_name: String,
pub input_dir: PathBuf,
pub output_dir: PathBuf,
pub time_bin_size: Nanos,
}
pub async fn convert(convert_params: ConvertParams) -> Result<(), Error> {
let _ = tokio::fs::create_dir(&convert_params.output_dir).await;
let meta = tokio::fs::metadata(&convert_params.output_dir).await?;
if !meta.is_dir() {
return Err(Error::from_string(format!(
"Given output path is not a directory: {:?}",
convert_params.output_dir
)));
}
let bs = convert_params.time_bin_size.clone();
let mut channel_config: Option<Config> = None;
let channel = Channel {
backend: String::new(),
name: convert_params.channel_name.into(),
series: None,
};
let mut data_writer = DataWriter::new(
convert_params.output_dir,
convert_params.keyspace_prefix.into(),
channel.clone(),
bs.clone(),
)
.await?;
let chandir = archapp::events::directory_for_channel_files(&channel, &convert_params.input_dir)?;
eprintln!("Looking for files in: {:?}", chandir);
let files = archapp::events::find_files_for_channel(&convert_params.input_dir, &channel).await?;
let mut evstot = 0;
for file in files {
eprintln!("Try to open {:?}", file);
let fni = archapp::events::parse_data_filename(file.to_str().unwrap())?;
debug!("fni: {:?}", fni);
let ts0 = Utc.ymd(fni.year as i32, fni.month, 1).and_hms(0, 0, 0);
let ts1 = ts0.timestamp() as u64 * SEC + ts0.timestamp_subsec_nanos() as u64;
let _ = ts1;
let mut f1 = File::open(&file).await?;
let _flen = f1.seek(SeekFrom::End(0)).await?;
f1.seek(SeekFrom::Start(0)).await?;
let pbr = PbFileReader::new(f1).await?;
eprintln!(
"PBR file header channel name: {:?} data type: {:?}",
pbr.channel_name(),
pbr.payload_type()
);
debug!("channel name in pbr file: {:?}", pbr.channel_name());
debug!("data type in file: {:?}", pbr.payload_type());
if pbr.channel_name() != channel.name() {
return Err(Error::with_msg(format!(
"channel name mismatch: {:?} vs {:?}",
pbr.channel_name(),
channel.name()
)));
}
let evq = RawEventsQuery::new(
channel.clone(),
NanoRange {
beg: u64::MIN,
end: u64::MAX,
},
AggKind::Plain,
);
let f1 = pbr.into_file();
// TODO can the positioning-logic maybe re-use the pbr?
let z = archapp::events::position_file_for_evq(f1, evq.clone(), fni.year).await?;
if let PositionState::Positioned(pos) = z.state {
let mut pbr = z.pbr;
assert_eq!(pos, pbr.abspos());
let mut i1 = 0;
let mut repnext = u64::MAX;
loop {
match pbr.read_msg().await {
Ok(Some(ei)) => {
use items::{WithLen, WithTimestamps};
let ei = ei.item;
if ei.is_wave() {
eprintln!("ERROR wave channels are not yet fully supported");
return Ok(());
}
if ei.len() > 0 {
let scalar_type = ei.scalar_type();
let shape = match &ei {
items::eventsitem::EventsItem::Plain(k) => match k.shape() {
Shape::Scalar => None,
Shape::Wave(n) => Some(vec![n]),
Shape::Image(..) => panic!(),
},
items::eventsitem::EventsItem::XBinnedEvents(_) => panic!(),
};
if let Some(conf) = &channel_config {
if scalar_type != conf.entries[0].scalar_type {
let msg = format!(
"unexpected type: {:?} vs {:?}",
scalar_type, conf.entries[0].scalar_type
);
return Err(Error::with_msg_no_trace(msg));
}
if shape != conf.entries[0].shape {
let msg = format!("unexpected shape: {:?} vs {:?}", shape, conf.entries[0].shape);
return Err(Error::with_msg_no_trace(msg));
}
}
if channel_config.is_none() {
let ks = if ei.is_wave() { 3 } else { 2 };
let e = parse::channelconfig::ConfigEntry {
ts: 0,
pulse: 0,
ks,
bs: bs.clone(),
split_count: 1,
status: 0,
bb: 0,
modulo: 0,
offset: 0,
precision: 0,
scalar_type: scalar_type,
is_compressed: false,
is_shaped: false,
is_array: false,
byte_order: netpod::ByteOrder::LE,
compression_method: None,
shape,
source_name: None,
unit: None,
description: None,
optional_fields: None,
value_converter: None,
};
let k = parse::channelconfig::Config {
format_version: 0,
channel_name: channel.name().into(),
entries: vec![e],
};
channel_config = Some(k);
}
match &ei {
items::eventsitem::EventsItem::Plain(item) => {
data_writer.write_item(item).await?;
}
items::eventsitem::EventsItem::XBinnedEvents(_) => {
panic!()
}
}
}
let tslast = if ei.len() > 0 { Some(ei.ts(ei.len() - 1)) } else { None };
if i1 == repnext {
debug!("read msg from file {} len {} tslast {:?}", i1, ei.len(), tslast);
repnext = 1 + 4 * repnext / 3;
}
i1 += 1;
if false {
ei.x_aggregate(&evq.agg_kind);
}
}
Ok(None) => {
debug!("reached end of file");
break;
}
Err(e) => {
error!("error while reading msg {:?}", e);
break;
}
}
}
debug!("read total {} events from the last file", i1);
evstot += i1;
} else {
error!("Position fail.");
}
}
eprintln!("Total number of events converted: {}", evstot);
data_writer.write_config(channel_config.as_ref().unwrap()).await?;
Ok(())
}
httpret/Cargo.toml
-1
View File
@@ -30,7 +30,6 @@ tokio-postgres = { version = "0.7.6", features = ["runtime", "with-chrono-0_4",
disk = { path = "../disk" }
items = { path = "../items" }
parse = { path = "../parse" }
archapp_wrap = { path = "../archapp_wrap" }
nodenet = { path = "../nodenet" }
commonio = { path = "../commonio" }
taskrun = { path = "../taskrun" }
httpret/src/channelarchiver.rs
-427
View File
@@ -1,427 +0,0 @@
use crate::err::Error;
use crate::response;
use futures_core::Stream;
use futures_util::{StreamExt, TryStreamExt};
use http::{header, Method, Request, Response, StatusCode};
use hyper::Body;
use netpod::log::*;
use netpod::{get_url_query_pairs, Channel, NanoRange};
use netpod::{NodeConfigCached, APP_JSON_LINES};
use serde::Serialize;
use serde_json::Value as JsVal;
use url::Url;
fn json_lines_stream<S, I>(stream: S) -> impl Stream<Item = Result<Vec<u8>, Error>>
where
S: Stream<Item = Result<I, Error>>,
I: Serialize,
{
stream.map(|k| {
k.map(|k| {
let mut a = serde_json::to_vec(&k).unwrap();
a.push(0xa);
a
})
})
}
pub struct ListIndexFilesHttpFunction {}
impl ListIndexFilesHttpFunction {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/list/indexfiles"
}
pub fn name() -> &'static str {
"ListIndexFilesHttpFunction"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::list_index_files(conf);
let s = futures_util::stream::unfold(s, |mut st| async move {
let x = st.next().await;
match x {
Some(x) => match x {
Ok(x) => {
let mut x = serde_json::to_vec(&x).unwrap();
x.push(b'\n');
Some((Ok::<_, Error>(x), st))
}
Err(e) => {
error!("{:?}", e);
None
}
},
None => None,
}
});
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct ScanIndexFiles {}
impl ScanIndexFiles {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/scan/indexfiles"
}
pub fn name() -> &'static str {
"ScanIndexFiles"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_index_files(conf.clone(), node_config.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct ScanChannels {}
impl ScanChannels {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/scan/channels"
}
pub fn name() -> &'static str {
"ScanChannels"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::indexfiles::scan_channels(node_config.clone(), conf.clone());
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct ChannelNames {}
impl ChannelNames {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/channel/names"
}
pub fn name() -> &'static str {
"ChannelNames"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let database = &node_config.node_config.cluster.database;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(database.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(stream))?)
}
}
pub struct ScanConfigs {}
impl ScanConfigs {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/scan/configs"
}
pub fn name() -> &'static str {
"ScanConfigs"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let database = &node_config.node_config.cluster.database;
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(database.clone());
let stream = archeng::configs::ConfigStream::new(stream, node_config.clone(), conf.clone());
let stream = stream.map_err(Error::from);
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(stream))?)
}
}
pub struct BlockRefStream {}
impl BlockRefStream {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/blockrefstream"
}
pub fn name() -> &'static str {
"BlockRefStream"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let database = &node_config.node_config.cluster.database;
let range = NanoRange { beg: 0, end: u64::MAX };
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let channel_name = pairs.get("channelName").map(String::from).unwrap_or("NONE".into());
let channel = Channel {
backend: "".into(),
name: channel_name,
series: None,
//name: "ARIDI-PCT:CURRENT".into(),
};
use archapp_wrap::archapp::archeng;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), database.clone()).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let s = archeng::blockrefstream::blockref_stream(channel, range, true, ixpath);
let s = s.map(|item| match item {
Ok(item) => {
use archeng::blockrefstream::BlockrefItem::*;
match item {
Blockref(_k, jsval) => Ok(jsval),
JsVal(jsval) => Ok(jsval),
}
}
Err(e) => Err(e),
});
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),
Err(e) => {
error!("observe error: {}", e);
Err(e)
}
});
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct BlockStream {}
impl BlockStream {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/blockstream"
}
pub fn name() -> &'static str {
"BlockStream"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let database = &node_config.node_config.cluster.database;
let range = NanoRange { beg: 0, end: u64::MAX };
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let read_queue = pairs.get("readQueue").unwrap_or(&"1".to_string()).parse()?;
let channel_name = pairs.get("channelName").map(String::from).unwrap_or("NONE".into());
let channel = Channel {
backend: node_config.node_config.cluster.backend.clone(),
name: channel_name,
series: None,
};
use archapp_wrap::archapp::archeng;
let ixpaths = archeng::indexfiles::index_file_path_list(channel.clone(), database.clone()).await?;
info!("got categorized ixpaths: {:?}", ixpaths);
let ixpath = ixpaths.first().unwrap().clone();
let s = archeng::blockrefstream::blockref_stream(channel, range.clone(), true, ixpath);
let s = Box::pin(s);
let s = archeng::blockstream::BlockStream::new(s, range.clone(), read_queue);
let s = s.map(|item| match item {
Ok(item) => {
use archeng::blockstream::BlockItem;
match item {
BlockItem::EventsItem(_item) => Ok(JsVal::String("EventsItem".into())),
BlockItem::JsVal(jsval) => Ok(jsval),
}
}
Err(e) => Err(e),
});
let s = s.map_err(Error::from);
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),
Err(e) => {
error!("observe error: {}", e);
Err(e)
}
});
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct ListChannelsHttpFunction {}
impl ListChannelsHttpFunction {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/list/channels"
}
pub fn name() -> &'static str {
"ListChannelsHttpFunction"
}
pub fn handler(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let s = archapp_wrap::archapp::archeng::list_all_channels(conf);
let s = futures_util::stream::unfold(s, |mut st| async move {
let x = st.next().await;
match x {
Some(x) => match x {
Ok(x) => {
let mut x = serde_json::to_vec(&x).unwrap();
x.push(b'\n');
Some((Ok::<_, Error>(x), st))
}
Err(e) => {
error!("{:?}", e);
None
}
},
None => None,
}
});
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
httpret/src/channelconfig.rs
+4 -5
View File
@@ -271,11 +271,10 @@ pub async fn channel_config(req: Request<Body>, node_config: &NodeConfigCached)
let conf = if let Some(scyco) = &node_config.node_config.cluster.scylla {
let pgconf = node_config.node_config.cluster.database.clone();
config_from_scylla(q, pgconf, scyco.clone(), node_config).await?
} else if let Some(conf) = &node_config.node.channel_archiver {
archapp_wrap::archapp::archeng::channel_config_from_db(&q, conf, &node_config.node_config.cluster.database)
.await?
} else if let Some(conf) = &node_config.node.archiver_appliance {
archapp_wrap::channel_config(&q, conf).await?
} else if let Some(_) = &node_config.node.channel_archiver {
return Err(Error::with_msg_no_trace("archapp not built"));
} else if let Some(_) = &node_config.node.archiver_appliance {
return Err(Error::with_msg_no_trace("archapp not built"));
} else {
parse::channelconfig::channel_config(&q, &node_config.node).await?
};
httpret/src/httpret.rs
-110
View File
@@ -1,6 +1,5 @@
pub mod api1;
pub mod bodystream;
pub mod channelarchiver;
pub mod channelconfig;
pub mod download;
pub mod err;
@@ -29,7 +28,6 @@ use net::SocketAddr;
use netpod::log::*;
use netpod::query::BinnedQuery;
use netpod::timeunits::SEC;
use netpod::{get_url_query_pairs, Channel};
use netpod::{FromUrl, NodeConfigCached, NodeStatus, NodeStatusArchiverAppliance};
use netpod::{ACCEPT_ALL, APP_JSON, APP_JSON_LINES, APP_OCTET};
use nodenet::conn::events_service;
@@ -295,24 +293,6 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/4/archapp/files/scan/msgs" {
if req.method() == Method::GET {
Ok(archapp_scan_files(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/4/archapp/files/scan/insert" {
if req.method() == Method::GET {
Ok(archapp_scan_files_insert(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if path == "/api/4/archapp/channel/info" {
if req.method() == Method::GET {
Ok(archapp_channel_info(req, &node_config).await?)
} else {
Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
}
} else if let Some(h) = download::DownloadHandler::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = settings::SettingsThreadsMaxHandler::handler(&req) {
@@ -335,22 +315,6 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = pulsemap::Api4MapPulseHttpFunction::handler(&req) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ListIndexFilesHttpFunction::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ListChannelsHttpFunction::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanIndexFiles::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanChannels::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanConfigs::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ChannelNames::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockRefStream::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockStream::handler(path) {
h.handle(req, &node_config).await
} else if let Some(h) = api1::RequestStatusHandler::handler(&req) {
h.handle(req, &node_config).await
} else if path.starts_with("/api/1/documentation/") {
@@ -826,77 +790,3 @@ pub fn status_board() -> Result<RwLockWriteGuard<'static, StatusBoard>, Error> {
Err(e) => Err(Error::with_msg(format!("{e:?}"))),
}
}
pub async fn archapp_scan_files(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let res = archapp_wrap::scan_files(pairs, node_config.clone()).await?;
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(res.map(|k| match k {
Ok(k) => match k.serialize() {
Ok(mut item) => {
item.push(0xa);
Ok(item)
}
Err(e) => Err(e),
},
Err(e) => match serde_json::to_vec(&e) {
Ok(mut item) => {
item.push(0xa);
Ok(item)
}
Err(e) => Err(e.into()),
},
})))?;
Ok(ret)
}
pub async fn archapp_scan_files_insert(
req: Request<Body>,
node_config: &NodeConfigCached,
) -> Result<Response<Body>, Error> {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let res = archapp_wrap::scan_files_insert(pairs, node_config.clone()).await?;
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(res.map(|k| match k {
Ok(k) => match k.serialize() {
Ok(mut item) => {
item.push(0xa);
Ok(item)
}
Err(e) => Err(e),
},
Err(e) => match serde_json::to_vec(&e) {
Ok(mut item) => {
item.push(0xa);
Ok(item)
}
Err(e) => Err(e.into()),
},
})))?;
Ok(ret)
}
pub async fn archapp_channel_info(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let channel = Channel::from_pairs(&pairs)?;
match archapp_wrap::channel_info(&channel, node_config).await {
Ok(res) => {
let buf = serde_json::to_vec(&res)?;
let ret = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(buf))?;
Ok(ret)
}
Err(e) => {
let ret = response(StatusCode::INTERNAL_SERVER_ERROR)
.header(http::header::CONTENT_TYPE, "text/text")
.body(Body::from(format!("{:?}", e)))?;
Ok(ret)
}
}
}
nodenet/Cargo.toml
-1
View File
@@ -28,7 +28,6 @@ tokio-postgres = "0.7.6"
err = { path = "../err" }
netpod = { path = "../netpod" }
disk = { path = "../disk" }
archapp_wrap = { path = "../archapp_wrap" }
#parse = { path = "../parse" }
items = { path = "../items" }
dbconn = { path = "../dbconn" }
nodenet/src/conn.rs
+6 -10
View File
@@ -156,16 +156,12 @@ async fn events_conn_handler_inner_try(
}
Err(e) => return Err((e, netout))?,
}
} else if let Some(aa) = &node_config.node.channel_archiver {
match archapp_wrap::archapp::archeng::pipe::make_event_pipe(&evq, node_config.clone(), aa.clone()).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else if let Some(aa) = &node_config.node.archiver_appliance {
match archapp_wrap::make_event_pipe(&evq, aa).await {
Ok(j) => j,
Err(e) => return Err((e, netout))?,
}
} else if let Some(_) = &node_config.node.channel_archiver {
let e = Error::with_msg_no_trace("archapp not built");
return Err((e, netout))?;
} else if let Some(_) = &node_config.node.archiver_appliance {
let e = Error::with_msg_no_trace("archapp not built");
return Err((e, netout))?;
} else {
match evq.agg_kind {
AggKind::EventBlobs => match disk::raw::conn::make_event_blobs_pipe(&evq, node_config).await {