CBOR chunked download

crates/httpret/src/api4/events.rs

@@ -3,8 +3,12 @@ use crate::err::Error;
 use crate::response;
 use crate::response_err;
 use crate::ToPublicResponse;
+use bytes::Bytes;
+use bytes::BytesMut;
+use futures_util::future;
 use futures_util::stream;
-use futures_util::TryStreamExt;
+use futures_util::StreamExt;
+use http::header;
 use http::Method;
 use http::StatusCode;
 use httpclient::body_empty;
@@ -19,8 +23,8 @@ use netpod::FromUrl;
 use netpod::NodeConfigCached;
 use netpod::ReqCtx;
 use netpod::ACCEPT_ALL;
+use netpod::APP_CBOR;
 use netpod::APP_JSON;
-use netpod::APP_OCTET;
 use query::api4::events::PlainEventsQuery;
 use url::Url;
 
@@ -58,32 +62,41 @@ async fn plain_events(req: Requ, ctx: &ReqCtx, node_config: &NodeConfigCached) -
     let accept_def = APP_JSON;
     let accept = req
         .headers()
-        .get(http::header::ACCEPT)
+        .get(header::ACCEPT)
         .map_or(accept_def, |k| k.to_str().unwrap_or(accept_def));
     let url = req_uri_to_url(req.uri())?;
     if accept.contains(APP_JSON) || accept.contains(ACCEPT_ALL) {
         Ok(plain_events_json(url, req, ctx, node_config).await?)
-    } else if accept == APP_OCTET {
-        Ok(plain_events_binary(url, req, ctx, node_config).await?)
+    } else if accept == APP_CBOR {
+        Ok(plain_events_cbor(url, req, ctx, node_config).await?)
     } else {
-        let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("Unsupported Accept: {:?}", accept))?;
+        let ret = response_err(StatusCode::NOT_ACCEPTABLE, format!("unsupported accept: {}", accept))?;
         Ok(ret)
     }
 }
 
-async fn plain_events_binary(
-    url: Url,
-    req: Requ,
-    ctx: &ReqCtx,
-    node_config: &NodeConfigCached,
-) -> Result<StreamResponse, Error> {
-    debug!("{:?}", req);
-    let query = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
-    let ch_conf = chconf_from_events_quorum(&query, ctx, node_config).await?;
-    info!("plain_events_binary  chconf_from_events_quorum: {ch_conf:?}");
-    let s = stream::iter([Ok::<_, Error>(String::from("TODO_PREBINNED_BINARY_STREAM"))]);
-    let s = s.map_err(Error::from);
-    let ret = response(StatusCode::OK).body(body_stream(s))?;
+async fn plain_events_cbor(url: Url, req: Requ, ctx: &ReqCtx, ncc: &NodeConfigCached) -> Result<StreamResponse, Error> {
+    let evq = PlainEventsQuery::from_url(&url).map_err(|e| e.add_public_msg(format!("Can not understand query")))?;
+    let ch_conf = chconf_from_events_quorum(&evq, ctx, ncc)
+        .await?
+        .ok_or_else(|| Error::with_msg_no_trace("channel not found"))?;
+    info!("plain_events_cbor  chconf_from_events_quorum: {ch_conf:?}  {req:?}");
+    let stream = streams::plaineventscbor::plain_events_cbor(&evq, ch_conf, ctx, ncc).await?;
+    use future::ready;
+    let stream = stream
+        .flat_map(|x| match x {
+            Ok(y) => {
+                use bytes::BufMut;
+                let buf = y.into_inner();
+                let mut b2 = BytesMut::with_capacity(8);
+                b2.put_u32_le(buf.len() as u32);
+                stream::iter([Ok::<_, Error>(b2.freeze()), Ok(buf)])
+            }
+            // TODO handle other cases
+            _ => stream::iter([Ok(Bytes::new()), Ok(Bytes::new())]),
+        })
+        .filter(|x| if let Ok(x) = x { ready(x.len() > 0) } else { ready(true) });
+    let ret = response(StatusCode::OK).body(body_stream(stream))?;
     Ok(ret)
 }
 

crates/httpret/src/proxy.rs

@@ -190,8 +190,8 @@ async fn proxy_http_service_inner(
         Ok(backends(req, proxy_config).await?)
     } else if let Some(h) = api4::ChannelSearchAggHandler::handler(&req) {
         h.handle(req, ctx, &proxy_config).await
-    } else if path == "/api/4/events" {
-        Ok(proxy_single_backend_query::<PlainEventsQuery>(req, ctx, proxy_config).await?)
+    } else if let Some(h) = api4::events::EventsHandler::handler(&req) {
+        h.handle(req, ctx, &proxy_config).await
     } else if path == "/api/4/status/connection/events" {
         Ok(proxy_single_backend_query::<ChannelStateEventsQuery>(req, ctx, proxy_config).await?)
     } else if path == "/api/4/status/channel/events" {
@@ -550,7 +550,7 @@ where
                         }
                         Err(e) => Err(Error::with_msg(format!("parse error for: {:?}  {:?}", sh, e))),
                     })
-                    .fold_ok(vec![], |mut a, x| {
+                    .fold_ok(Vec::new(), |mut a, x| {
                         a.push(x);
                         a
                     })?;
@@ -631,3 +631,15 @@ fn get_query_host_for_backend(backend: &str, proxy_config: &ProxyConfig) -> Resu
     }
     return Err(Error::with_msg(format!("host not found for backend {:?}", backend)));
 }
+
+fn get_random_query_host_for_backend(backend: &str, proxy_config: &ProxyConfig) -> Result<String, Error> {
+    let url = get_query_host_for_backend(backend, proxy_config)?;
+    // TODO remove special code, make it part of configuration
+    if url.contains("sf-daqbuf-23.psi.ch") {
+        let id = 21 + rand::random::<u32>() % 13;
+        let url = url.replace("-23.", &format!("-{id}."));
+        Ok(url)
+    } else {
+        Ok(url)
+    }
+}

crates/httpret/src/proxy/api4.rs

@@ -1,4 +1,5 @@
 pub mod caioclookup;
+pub mod events;
 
 use crate::bodystream::ToPublicResponse;
 use crate::err::Error;

crates/httpret/src/proxy/api4/events.rs

@@ -0,0 +1,101 @@
+use crate::bodystream::response;
+use crate::err::Error;
+use crate::proxy::get_query_host_for_backend;
+use crate::ReqCtx;
+use http::header;
+use http::HeaderValue;
+use http::Method;
+use http::Request;
+use http::StatusCode;
+use http::Uri;
+use httpclient::body_bytes;
+use httpclient::body_empty;
+use httpclient::body_stream;
+use httpclient::connect_client;
+use httpclient::read_body_bytes;
+use httpclient::Requ;
+use httpclient::StreamIncoming;
+use httpclient::StreamResponse;
+use netpod::get_url_query_pairs;
+use netpod::log::*;
+use netpod::query::api1::Api1Query;
+use netpod::req_uri_to_url;
+use netpod::FromUrl;
+use netpod::HasBackend;
+use netpod::ProxyConfig;
+use netpod::ACCEPT_ALL;
+use netpod::APP_CBOR;
+use netpod::APP_JSON;
+use netpod::X_DAQBUF_REQID;
+use query::api4::events::PlainEventsQuery;
+
+pub struct EventsHandler {}
+
+impl EventsHandler {
+    pub fn path() -> &'static str {
+        "/api/4/events"
+    }
+
+    pub fn handler(req: &Requ) -> Option<Self> {
+        if req.uri().path() == Self::path() {
+            Some(Self {})
+        } else {
+            None
+        }
+    }
+
+    pub async fn handle(&self, req: Requ, ctx: &ReqCtx, proxy_config: &ProxyConfig) -> Result<StreamResponse, Error> {
+        if req.method() != Method::GET {
+            return Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(body_empty())?);
+        }
+
+        let def = HeaderValue::from_static("*/*");
+        let accept = req.headers().get(header::ACCEPT).unwrap_or(&def);
+        let accept = accept.to_str()?;
+
+        if accept.contains(APP_CBOR) {
+            self.handle_cbor(req, ctx, proxy_config).await
+        } else if accept.contains(APP_JSON) {
+            return Ok(crate::proxy::proxy_single_backend_query::<PlainEventsQuery>(req, ctx, proxy_config).await?);
+        } else if accept.contains(ACCEPT_ALL) {
+            todo!()
+        } else {
+            return Err(Error::with_msg_no_trace(format!("bad accept {:?}", req.headers())));
+        }
+    }
+
+    async fn handle_cbor(&self, req: Requ, ctx: &ReqCtx, proxy_config: &ProxyConfig) -> Result<StreamResponse, Error> {
+        let (head, _body) = req.into_parts();
+        let url = req_uri_to_url(&head.uri)?;
+        let pairs = get_url_query_pairs(&url);
+        let evq = PlainEventsQuery::from_pairs(&pairs)?;
+        info!("{evq:?}");
+        // Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body_empty())?)
+
+        // get_random_query_host_for_backend
+        let query_host = get_query_host_for_backend(evq.backend(), proxy_config)?;
+
+        // TODO this ignores the fragment
+        let url_str = format!("{}{}", query_host, head.uri.path_and_query().unwrap());
+        info!("try to contact {url_str}");
+
+        let uri: Uri = url_str.parse()?;
+        let req = Request::builder()
+            .method(Method::GET)
+            .header(header::HOST, uri.host().unwrap())
+            .header(header::ACCEPT, APP_CBOR)
+            .header(ctx.header_name(), ctx.header_value())
+            .uri(&uri)
+            .body(body_empty())?;
+        let mut client = connect_client(&uri).await?;
+        let res = client.send_request(req).await?;
+        let (head, body) = res.into_parts();
+        if head.status != StatusCode::OK {
+            error!("backend returned error: {head:?}");
+            Ok(response(StatusCode::INTERNAL_SERVER_ERROR).body(body_empty())?)
+        } else {
+            info!("backend returned OK");
+            Ok(response(StatusCode::OK).body(body_stream(StreamIncoming::new(body)))?)
+        }
+    }
+}