daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test on the VM with NFS confirms again low iops

Browse Source
This commit is contained in:
Dominik Werder
2021-11-02 09:08:58 +01:00
parent f48e4b8ea4
commit 4d3965660c
8 changed files with 707 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
httpret/src/channelarchiver.rs
View File

@@ -1,12 +1,15 @@
use crate::response;
use disk::events::PlainEventsJsonQuery;
use err::Error;
use futures_core::Stream;
use futures_util::StreamExt;
use http::{header, Method, Request, Response, StatusCode};
use hyper::Body;
use netpod::{log::*, Channel, NanoRange};
use netpod::query::RawEventsQuery;
use netpod::{get_url_query_pairs, log::*, Channel, NanoRange};
use netpod::{NodeConfigCached, APP_JSON_LINES};
use serde::Serialize;
use url::Url;
fn json_lines_stream<S, I>(stream: S) -> impl Stream<Item = Result<Vec<u8>, Error>>
where
@@ -155,6 +158,67 @@ impl ScanChannels {
}
}
pub struct BlockRefStream {}
impl BlockRefStream {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/blockrefstream"
}
pub fn name() -> &'static str {
"BlockRefStream"
}
pub fn should_handle(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
let range = NanoRange { beg: 0, end: u64::MAX };
let channel = Channel {
backend: "".into(),
name: "ARIDI-PCT:CURRENT".into(),
};
let s = archapp_wrap::archapp::archeng::blockrefstream::blockref_stream(channel, range, conf.clone());
let s = s.map(|item| match item {
Ok(item) => {
use archapp_wrap::archapp::archeng::blockrefstream::BlockrefItem::*;
match item {
Blockref(_k, jsval) => Ok(jsval),
JsVal(jsval) => Ok(jsval),
}
}
Err(e) => Err(e),
});
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),
Err(e) => {
error!("observe error: {}", e);
Err(e)
}
});
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(s))?)
}
}
pub struct BlockStream {}
impl BlockStream {
@@ -191,7 +255,19 @@ impl BlockStream {
backend: "".into(),
name: "ARIDI-PCT:CURRENT".into(),
};
let s = archapp_wrap::archapp::archeng::blockstream::blockstream(channel, range, conf.clone());
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let read_queue = pairs.get("readQueue").unwrap_or(&"1".to_string()).parse()?;
let s = archapp_wrap::archapp::archeng::blockrefstream::blockref_stream(channel, range.clone(), conf.clone());
let s = Box::pin(s);
let s = archapp_wrap::archapp::archeng::blockstream::BlockStream::new(s, range.clone(), read_queue);
let s = s.map(|item| match item {
Ok(item) => {
//use archapp_wrap::archapp::archeng::blockstream::BlockItem::*;
Ok(item)
}
Err(e) => Err(e),
});
let s = json_lines_stream(s);
let s = s.map(|item| match item {
Ok(k) => Ok(k),

2
httpret/src/lib.rs
View File

@@ -290,6 +290,8 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanChannels::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockRefStream::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockStream::should_handle(path) {
h.handle(req, &node_config).await
} else if path.starts_with("/api/1/requestStatus/") {