daqbuf/daqbuffer
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve search api

Browse Source
This commit is contained in:
Dominik Werder
2021-11-03 16:00:53 +01:00
parent 4d3965660c
commit 96e0473392
21 changed files with 1285 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
httpret/src/api1.rs
View File

@@ -99,10 +99,12 @@ pub async fn channel_search_list_v1(req: Request<Body>, proxy_config: &ProxyConf
let (head, reqbody) = req.into_parts();
let bodybytes = hyper::body::to_bytes(reqbody).await?;
let query: ChannelSearchQueryV1 = serde_json::from_slice(&bodybytes)?;
match head.headers.get("accept") {
match head.headers.get(http::header::ACCEPT) {
Some(v) => {
if v == APP_JSON {
let query = ChannelSearchQuery {
// TODO
backend: None,
name_regex: query.regex.map_or(String::new(), |k| k),
source_regex: query.source_regex.map_or(String::new(), |k| k),
description_regex: query.description_regex.map_or(String::new(), |k| k),
@@ -190,11 +192,13 @@ pub async fn channel_search_configs_v1(
let (head, reqbody) = req.into_parts();
let bodybytes = hyper::body::to_bytes(reqbody).await?;
let query: ChannelSearchQueryV1 = serde_json::from_slice(&bodybytes)?;
match head.headers.get("accept") {
match head.headers.get(http::header::ACCEPT) {
Some(v) => {
if v == APP_JSON {
// Transform the ChannelSearchQueryV1 to ChannelSearchQuery
let query = ChannelSearchQuery {
// TODO
backend: None,
name_regex: query.regex.map_or(String::new(), |k| k),
source_regex: query.source_regex.map_or(String::new(), |k| k),
description_regex: query.description_regex.map_or(String::new(), |k| k),

110
httpret/src/channelarchiver.rs
View File

@@ -9,6 +9,7 @@ use netpod::query::RawEventsQuery;
use netpod::{get_url_query_pairs, log::*, Channel, NanoRange};
use netpod::{NodeConfigCached, APP_JSON_LINES};
use serde::Serialize;
use serde_json::Value as JsVal;
use url::Url;
fn json_lines_stream<S, I>(stream: S) -> impl Stream<Item = Result<Vec<u8>, Error>>
@@ -158,6 +159,87 @@ impl ScanChannels {
}
}
pub struct ChannelNames {}
impl ChannelNames {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/channel/names"
}
pub fn name() -> &'static str {
"ChannelNames"
}
pub fn should_handle(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(stream))?)
}
}
pub struct ScanConfigs {}
impl ScanConfigs {
pub fn prefix() -> &'static str {
"/api/4/channelarchiver/scan/configs"
}
pub fn name() -> &'static str {
"ScanConfigs"
}
pub fn should_handle(path: &str) -> Option<Self> {
if path.starts_with(Self::prefix()) {
Some(Self {})
} else {
None
}
}
pub async fn handle(&self, req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
if req.method() != Method::GET {
return Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?);
}
info!("{} handle uri: {:?}", Self::name(), req.uri());
let conf = node_config
.node
.channel_archiver
.as_ref()
.ok_or(Error::with_msg_no_trace(
"this node is not configured as channel archiver",
))?;
use archapp_wrap::archapp::archeng;
let stream = archeng::configs::ChannelNameStream::new(conf.database.clone());
let stream = archeng::configs::ConfigStream::new(stream, conf.clone());
let stream = json_lines_stream(stream);
Ok(response(StatusCode::OK)
.header(header::CONTENT_TYPE, APP_JSON_LINES)
.body(Body::wrap_stream(stream))?)
}
}
pub struct BlockRefStream {}
impl BlockRefStream {
@@ -190,9 +272,13 @@ impl BlockRefStream {
"this node is not configured as channel archiver",
))?;
let range = NanoRange { beg: 0, end: u64::MAX };
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let channel_name = pairs.get("channelName").map(String::from).unwrap_or("NONE".into());
let channel = Channel {
backend: "".into(),
name: "ARIDI-PCT:CURRENT".into(),
name: channel_name,
//name: "ARIDI-PCT:CURRENT".into(),
};
let s = archapp_wrap::archapp::archeng::blockrefstream::blockref_stream(channel, range, conf.clone());
let s = s.map(|item| match item {
@@ -251,20 +337,26 @@ impl BlockStream {
"this node is not configured as channel archiver",
))?;
let range = NanoRange { beg: 0, end: u64::MAX };
let channel = Channel {
backend: "".into(),
name: "ARIDI-PCT:CURRENT".into(),
};
let url = Url::parse(&format!("dummy:{}", req.uri()))?;
let pairs = get_url_query_pairs(&url);
let read_queue = pairs.get("readQueue").unwrap_or(&"1".to_string()).parse()?;
let s = archapp_wrap::archapp::archeng::blockrefstream::blockref_stream(channel, range.clone(), conf.clone());
let channel_name = pairs.get("channelName").map(String::from).unwrap_or("NONE".into());
let channel = Channel {
backend: "".into(),
name: channel_name,
//name: "ARIDI-PCT:CURRENT".into(),
};
use archapp_wrap::archapp::archeng;
let s = archeng::blockrefstream::blockref_stream(channel, range.clone(), conf.clone());
let s = Box::pin(s);
let s = archapp_wrap::archapp::archeng::blockstream::BlockStream::new(s, range.clone(), read_queue);
let s = archeng::blockstream::BlockStream::new(s, range.clone(), read_queue);
let s = s.map(|item| match item {
Ok(item) => {
//use archapp_wrap::archapp::archeng::blockstream::BlockItem::*;
Ok(item)
use archeng::blockstream::BlockItem;
match item {
BlockItem::EventsItem(item) => Ok(JsVal::String("EventsItem".into())),
BlockItem::JsVal(jsval) => Ok(jsval),
}
}
Err(e) => Err(e),
});

4
httpret/src/lib.rs
View File

@@ -290,6 +290,10 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanChannels::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ScanConfigs::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::ChannelNames::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockRefStream::should_handle(path) {
h.handle(req, &node_config).await
} else if let Some(h) = channelarchiver::BlockStream::should_handle(path) {

5
httpret/src/proxy.rs
View File

@@ -1,3 +1,5 @@
pub mod api4;
use crate::api1::{channel_search_configs_v1, channel_search_list_v1, gather_json_2_v1, proxy_distribute_v1};
use crate::gather::{gather_get_json_generic, SubRes};
use crate::{api_1_docs, api_4_docs, response, Cont};
@@ -78,7 +80,8 @@ async fn proxy_http_service_try(req: Request<Body>, proxy_config: &ProxyConfig)
} else if path == "/api/4/backends" {
Ok(backends(req, proxy_config).await?)
} else if path == "/api/4/search/channel" {
Ok(channel_search(req, proxy_config).await?)
//Ok(channel_search(req, proxy_config).await?)
Ok(api4::channel_search(req, proxy_config).await?)
} else if path == "/api/4/events" {
Ok(proxy_single_backend_query::<PlainEventsJsonQuery>(req, proxy_config).await?)
} else if path == "/api/4/binned" {

89
httpret/src/proxy/api4.rs Normal file
View File

@@ -0,0 +1,89 @@
use crate::gather::{gather_get_json_generic, SubRes};
use crate::response;
use err::Error;
use futures_core::Future;
use http::{header, Request, Response, StatusCode};
use hyper::Body;
use itertools::Itertools;
use netpod::log::*;
use netpod::{ChannelSearchQuery, ChannelSearchResult, ProxyConfig, APP_JSON};
use std::pin::Pin;
use std::time::Duration;
use url::Url;
pub async fn channel_search(req: Request<Body>, proxy_config: &ProxyConfig) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
let vdef = header::HeaderValue::from_static(APP_JSON);
let v = head.headers.get(header::ACCEPT).unwrap_or(&vdef);
if v == APP_JSON || v == "*/*" {
let inpurl = Url::parse(&format!("dummy:{}", head.uri))?;
let query = ChannelSearchQuery::from_url(&inpurl)?;
let mut bodies = vec![];
let urls = proxy_config
.backends
.iter()
.filter(|k| {
if let Some(back) = &query.backend {
back == &k.name
} else {
true
}
})
.map(|pb| match Url::parse(&format!("{}/api/4/search/channel", pb.url)) {
Ok(mut url) => {
query.append_to_url(&mut url);
Ok(url)
}
Err(_) => Err(Error::with_msg(format!("parse error for: {:?}", pb))),
})
.fold_ok(vec![], |mut a, x| {
a.push(x);
bodies.push(None);
a
})?;
let tags = urls.iter().map(|k| k.to_string()).collect();
let nt = |res| {
let fut = async {
let body = hyper::body::to_bytes(res).await?;
//info!("got a result {:?}", body);
let res: ChannelSearchResult = match serde_json::from_slice(&body) {
Ok(k) => k,
Err(_) => {
let msg = format!("can not parse result: {}", String::from_utf8_lossy(&body));
error!("{}", msg);
return Err(Error::with_msg_no_trace(msg));
}
};
Ok(res)
};
Box::pin(fut) as Pin<Box<dyn Future<Output = _> + Send>>
};
let ft = |all: Vec<SubRes<ChannelSearchResult>>| {
let mut res = vec![];
for j in all {
for k in j.val.channels {
res.push(k);
}
}
let res = ChannelSearchResult { channels: res };
let res = response(StatusCode::OK)
.header(http::header::CONTENT_TYPE, APP_JSON)
.body(Body::from(serde_json::to_string(&res)?))?;
Ok(res)
};
let ret = gather_get_json_generic(
http::Method::GET,
urls,
bodies,
tags,
nt,
ft,
Duration::from_millis(3000),
)
.await?;
Ok(ret)
} else {
info!("bad accept: {:?}", head.headers.get(header::ACCEPT));
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::from(format!("{:?}", proxy_config.name)))?)
}
}

27
httpret/src/search.rs
View File

@@ -8,18 +8,19 @@ use url::Url;
pub async fn channel_search(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
let (head, _body) = req.into_parts();
match head.headers.get(header::ACCEPT) {
Some(v) if v == APP_JSON => {
let s1 = format!("dummy:{}", head.uri);
info!("try to parse {:?}", s1);
let url = Url::parse(&s1)?;
let query = ChannelSearchQuery::from_url(&url)?;
info!("search query: {:?}", query);
let res = dbconn::search::search_channel(query, node_config).await?;
let body = Body::from(serde_json::to_string(&res)?);
let ret = super::response(StatusCode::OK).body(body)?;
Ok(ret)
}
_ => Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?),
let vdef = header::HeaderValue::from_static(APP_JSON);
let v = head.headers.get(header::ACCEPT).unwrap_or(&vdef);
if v == APP_JSON || v == "*/*" {
let s1 = format!("dummy:{}", head.uri);
info!("try to parse {:?}", s1);
let url = Url::parse(&s1)?;
let query = ChannelSearchQuery::from_url(&url)?;
info!("search query: {:?}", query);
let res = dbconn::search::search_channel(query, node_config).await?;
let body = Body::from(serde_json::to_string(&res)?);
let ret = super::response(StatusCode::OK).body(body)?;
Ok(ret)
} else {
Ok(response(StatusCode::NOT_ACCEPTABLE).body(Body::empty())?)
}
}