daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Dominik Werder
2023-06-20 19:00:08 +02:00
parent 1b3267c1a1
commit 2d566cc9ca
11 changed files with 452 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
daqbufp2/Cargo.toml
View File

@@ -32,6 +32,4 @@ disk = { path = "../disk" }
items_0 = { path = "../items_0" }
items_2 = { path = "../items_2" }
streams = { path = "../streams" }
[dev-dependencies]
nom = "7.1.1"
parse = { path = "../parse" }

21
daqbufp2/src/test/api1.rs
View File

@@ -1,16 +1,17 @@
mod api1_parse;
mod data_api_python;
use crate::nodes::require_test_hosts_running;
use crate::test::api1::api1_parse::Api1Frame;
use err::Error;
use futures_util::Future;
use httpclient::http_post;
use httpret::api1::Api1ScalarType;
use netpod::log::*;
use netpod::query::api1::Api1Query;
use netpod::query::api1::Api1Range;
use netpod::query::api1::ChannelTuple;
use netpod::APP_OCTET;
use parse::api1_parse;
use parse::api1_parse::Api1Frame;
use parse::api1_parse::Api1ScalarType;
use std::fmt;
use url::Url;
@@ -23,7 +24,7 @@ where
taskrun::run(fut)
}
fn is_monitonic_strict<I>(it: I) -> bool
fn is_monotonic_strict<I>(it: I) -> bool
where
I: Iterator,
<I as Iterator>::Item: PartialOrd + fmt::Debug,
@@ -42,8 +43,8 @@ where
#[test]
fn test_is_monitonic_strict() {
assert_eq!(is_monitonic_strict([1, 2, 3].iter()), true);
assert_eq!(is_monitonic_strict([1, 2, 2].iter()), false);
assert_eq!(is_monotonic_strict([1, 2, 3].iter()), true);
assert_eq!(is_monotonic_strict([1, 2, 2].iter()), false);
}
#[test]
@@ -57,7 +58,7 @@ fn events_f64_plain() -> Result<(), Error> {
let cluster = &rh.cluster;
let node = &cluster.nodes[0];
let url: Url = format!("http://{}:{}/api/1/query", node.host, node.port).parse()?;
let accept = "application/octet-stream";
let accept = APP_OCTET;
let range = Api1Range::new("1970-01-01T00:00:00Z".try_into()?, "1970-01-01T00:01:00Z".try_into()?)?;
// TODO the channel list needs to get pre-processed to check for backend prefix!
let ch = ChannelTuple::new(TEST_BACKEND.into(), "test-gen-i32-dim0-v01".into());
@@ -101,9 +102,9 @@ fn events_f64_plain() -> Result<(), Error> {
_ => None,
})
.collect();
assert_eq!(is_monitonic_strict(tss.iter()), true);
assert_eq!(is_monitonic_strict(pulses.iter()), true);
assert_eq!(is_monitonic_strict(values.iter()), true);
assert_eq!(is_monotonic_strict(tss.iter()), true);
assert_eq!(is_monotonic_strict(pulses.iter()), true);
assert_eq!(is_monotonic_strict(values.iter()), true);
for &val in &values {
assert!(val >= 0);
assert!(val < 120);

184
daqbufp2/src/test/api1/api1_parse.rs
View File

@@ -1,184 +0,0 @@
use httpret::api1::Api1ChannelHeader;
use netpod::log::*;
use nom::multi::many0;
use nom::number::complete::{be_u32, be_u64, be_u8};
use nom::IResult;
use std::num::NonZeroUsize;
// u32be length_1.
// there is exactly length_1 more bytes in this message.
// u8 mtype: 0: channel-header, 1: data
// for mtype == 0:
// The rest is a JSON with the channel header.
// for mtype == 1:
// u64be timestamp
// u64be pulse
// After that comes exactly (length_1 - 17) bytes of data.
#[derive(Debug)]
pub struct Header {
header: Api1ChannelHeader,
}
impl Header {
pub fn header(&self) -> &Api1ChannelHeader {
&self.header
}
}
#[derive(Debug)]
pub struct Data {
ts: u64,
pulse: u64,
data: Vec<u8>,
}
impl Data {
pub fn ts(&self) -> u64 {
self.ts
}
pub fn pulse(&self) -> u64 {
self.pulse
}
pub fn data(&self) -> &[u8] {
&self.data
}
}
#[derive(Debug)]
pub enum Api1Frame {
Header(Header),
Data(Data),
}
fn header(inp: &[u8]) -> IResult<&[u8], Header> {
match serde_json::from_slice(inp) {
Ok(k) => {
let k: Api1ChannelHeader = k;
IResult::Ok((&inp[inp.len()..], Header { header: k }))
}
Err(e) => {
error!("json header parse error: {e}");
let e = nom::Err::Failure(nom::error::make_error(inp, nom::error::ErrorKind::Fail));
IResult::Err(e)
}
}
}
fn data(inp: &[u8]) -> IResult<&[u8], Data> {
if inp.len() < 16 {
use nom::{Err, Needed};
return IResult::Err(Err::Incomplete(Needed::Size(NonZeroUsize::new(16).unwrap())));
}
let (inp, ts) = be_u64(inp)?;
let (inp, pulse) = be_u64(inp)?;
let data = inp.into();
let inp = &inp[inp.len()..];
let res = Data { ts, pulse, data };
IResult::Ok((inp, res))
}
fn api1_frame_complete(inp: &[u8]) -> IResult<&[u8], Api1Frame> {
let (inp, mtype) = be_u8(inp)?;
if mtype == 0 {
let (inp, val) = header(inp)?;
if inp.len() != 0 {
// We did not consume the exact number of bytes
let kind = nom::error::ErrorKind::Verify;
let e = nom::error::Error::new(inp, kind);
Err(nom::Err::Failure(e))
} else {
let res = Api1Frame::Header(val);
IResult::Ok((inp, res))
}
} else if mtype == 1 {
let (inp, val) = data(inp)?;
if inp.len() != 0 {
// We did not consume the exact number of bytes
let kind = nom::error::ErrorKind::Verify;
let e = nom::error::Error::new(inp, kind);
Err(nom::Err::Failure(e))
} else {
let res = Api1Frame::Data(val);
IResult::Ok((inp, res))
}
} else {
let e = nom::Err::Incomplete(nom::Needed::Size(NonZeroUsize::new(1).unwrap()));
IResult::Err(e)
}
}
fn api1_frame(inp: &[u8]) -> IResult<&[u8], Api1Frame> {
let (inp, len) = be_u32(inp)?;
if len < 1 {
use nom::error::{ErrorKind, ParseError};
use nom::Err;
return IResult::Err(Err::Failure(ParseError::from_error_kind(inp, ErrorKind::Fail)));
}
if inp.len() < len as usize {
let e = nom::Err::Incomplete(nom::Needed::Size(NonZeroUsize::new(len as _).unwrap()));
IResult::Err(e)
} else {
let (inp2, inp) = inp.split_at(len as _);
let (inp2, res) = api1_frame_complete(inp2)?;
if inp2.len() != 0 {
let kind = nom::error::ErrorKind::Fail;
let e = nom::error::Error::new(inp, kind);
IResult::Err(nom::Err::Failure(e))
} else {
IResult::Ok((inp, res))
}
}
}
type Nres<'a, T> = IResult<&'a [u8], T, nom::error::VerboseError<&'a [u8]>>;
#[allow(unused)]
fn verbose_err(inp: &[u8]) -> Nres<u32> {
use nom::error::{ErrorKind, ParseError, VerboseError};
use nom::Err;
let e = VerboseError::from_error_kind(inp, ErrorKind::Fail);
return IResult::Err(Err::Failure(e));
}
pub fn api1_frames(inp: &[u8]) -> IResult<&[u8], Vec<Api1Frame>> {
let (inp, res) = many0(api1_frame)(inp)?;
IResult::Ok((inp, res))
}
#[test]
fn test_basic_frames() -> Result<(), err::Error> {
use std::io::Write;
let mut buf = Vec::new();
let js = r#"{"name": "ch1", "type": "float64", "byteOrder": "LITTLE_ENDIAN"}"#;
buf.write(&(1 + js.as_bytes().len() as u32).to_be_bytes())?;
buf.write(&[0])?;
buf.write(js.as_bytes())?;
buf.write(&25u32.to_be_bytes())?;
buf.write(&[1])?;
buf.write(&20u64.to_be_bytes())?;
buf.write(&21u64.to_be_bytes())?;
buf.write(&5.123f64.to_be_bytes())?;
buf.write(&25u32.to_be_bytes())?;
buf.write(&[1])?;
buf.write(&22u64.to_be_bytes())?;
buf.write(&23u64.to_be_bytes())?;
buf.write(&7.88f64.to_be_bytes())?;
match api1_frames(&buf) {
Ok((_, frames)) => {
assert_eq!(frames.len(), 3);
}
Err(e) => {
error!("can not parse result: {e}");
panic!()
}
};
Ok(())
}

7
daqbufp2/src/test/api1/data_api_python.rs
View File

@@ -11,6 +11,7 @@ use netpod::HostPort;
use netpod::SfDbChannel;
use netpod::APP_JSON;
use netpod::DATETIME_FMT_3MS;
use parse::api1_parse::api1_frames;
use url::Url;
const TEST_BACKEND: &str = "testbackend-00";
@@ -74,6 +75,12 @@ fn api3_hdf_dim0_00() -> Result<(), Error> {
cluster,
)
.await?;
use parse::nom;
let x: Result<(&[u8], Vec<parse::api1_parse::Api1Frame>), nom::Err<nom::error::VerboseError<&[u8]>>> =
api1_frames(&jsv);
let res = x.unwrap();
eprintln!("{res:?}");
panic!();
Ok(())
};
taskrun::run(fut)