Move API 1 docs here

dbconn/src/lib.rs

@@ -33,20 +33,6 @@ pub async fn create_connection(node_config: &NodeConfigCached) -> Result<Client,
     Ok(cl)
 }
 
-pub async fn create_connection_2(node_config: NodeConfigCached) -> Result<Client, Error> {
-    let d = &node_config.node_config.cluster.database;
-    let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, 5432, d.name);
-    let (cl, conn) = tokio_postgres::connect(&uri, NoTls).await?;
-    // TODO monitor connection drop.
-    let _cjh = tokio::spawn(async move {
-        if let Err(e) = conn.await {
-            error!("connection error: {}", e);
-        }
-        Ok::<_, Error>(())
-    });
-    Ok(cl)
-}
-
 pub async fn channel_exists(channel: &Channel, node_config: &NodeConfigCached) -> Result<bool, Error> {
     let cl = create_connection(node_config).await?;
     let rows = cl

dbconn/src/scan.rs

@@ -1,4 +1,4 @@
-use async_channel::{bounded, Receiver, Sender};
+use async_channel::{bounded, Receiver};
 use chrono::{DateTime, Utc};
 use err::Error;
 use futures_core::Stream;
@@ -8,11 +8,9 @@ use netpod::NodeConfigCached;
 use serde::{Deserialize, Serialize};
 use std::future::Future;
 use std::io::ErrorKind;
-use std::marker::PhantomPinned;
 use std::os::unix::ffi::OsStringExt;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
-use std::ptr::NonNull;
 use std::sync::{Arc, RwLock};
 use std::task::{Context, Poll};
 use tokio_postgres::Client;
@@ -80,12 +78,12 @@ pub struct UpdatedDbWithChannelNames {
 pub struct UpdatedDbWithChannelNamesStream {
     errored: bool,
     data_complete: bool,
+    #[allow(dead_code)]
     node_config: Pin<Box<NodeConfigCached>>,
-    node_config_ptr: NonNull<NodeConfigCached>,
+    node_config_ref: &'static NodeConfigCached,
-    //_pin: PhantomPinned,
     client_fut: Option<Pin<Box<dyn Future<Output = Result<Client, Error>> + Send>>>,
     client: Option<Pin<Box<Client>>>,
-    client_ptr: *const Client,
+    client_ref: Option<&'static Client>,
     ident_fut: Option<Pin<Box<dyn Future<Output = Result<NodeDiskIdent, Error>> + Send>>>,
     ident: Option<NodeDiskIdent>,
 }
@@ -95,22 +93,19 @@ unsafe impl Send for UpdatedDbWithChannelNamesStream {}
 impl UpdatedDbWithChannelNamesStream {
     pub fn new(node_config: NodeConfigCached) -> Result<Self, Error> {
         let node_config = Box::pin(node_config.clone());
+        let node_config_ref = unsafe { &*(&node_config as &NodeConfigCached as *const _) };
         let mut ret = Self {
             errored: false,
             data_complete: false,
-            node_config_ptr: NonNull::dangling(),
             node_config,
-            //_pin: PhantomPinned,
+            node_config_ref,
             client_fut: None,
             client: None,
-            client_ptr: std::ptr::null(),
+            client_ref: None,
             ident_fut: None,
             ident: None,
         };
-        ret.node_config_ptr = NonNull::from(&*ret.node_config);
+        ret.client_fut = Some(Box::pin(crate::create_connection(ret.node_config_ref)));
-        ret.client_fut = Some(Box::pin(crate::create_connection(unsafe {
-            &*ret.node_config_ptr.as_ptr()
-        })));
         Ok(ret)
     }
 }
@@ -120,7 +115,7 @@ impl Stream for UpdatedDbWithChannelNamesStream {
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
         use Poll::*;
-        'outer: loop {
+        loop {
             break if self.errored {
                 Ready(None)
             } else if self.data_complete {
@@ -148,11 +143,10 @@ impl Stream for UpdatedDbWithChannelNamesStream {
                     Ready(Ok(item)) => {
                         self.client_fut = None;
                         self.client = Some(Box::pin(item));
-                        self.client_ptr = self.client.as_ref().unwrap() as &Client as *const _;
+                        self.client_ref = Some(unsafe { &*(&self.client.as_ref().unwrap() as &Client as *const _) });
-                        let p1 = unsafe { &*self.client_ptr };
                         self.ident_fut = Some(Box::pin(get_node_disk_ident(
-                            unsafe { &*self.node_config_ptr.as_ptr() },
+                            self.node_config_ref,
-                            &p1,
+                            self.client_ref.as_ref().unwrap(),
                         )));
                         let ret = UpdatedDbWithChannelNames {
                             msg: format!("Client opened connection"),
@@ -209,7 +203,7 @@ pub async fn update_db_with_channel_names(
     })
     .await?;
     dbc.query("commit", &[]).await?;
-    let ret = UpdatedDbWithChannelNames {
+    let _ret = UpdatedDbWithChannelNames {
         msg: format!("done"),
         count: *c1.read()?,
     };

httpret/Cargo.toml

@@ -20,4 +20,5 @@ err = { path = "../err" }
 netpod = { path = "../netpod" }
 dbconn = { path = "../dbconn" }
 disk = { path = "../disk" }
+parse = { path = "../parse" }
 taskrun = { path = "../taskrun" }

httpret/src/lib.rs

@@ -12,7 +12,7 @@ use hyper::service::{make_service_fn, service_fn};
 use hyper::{server::Server, Body, Request, Response};
 use net::SocketAddr;
 use netpod::log::*;
-use netpod::NodeConfigCached;
+use netpod::{Channel, NodeConfigCached};
 use panic::{AssertUnwindSafe, UnwindSafe};
 use pin::Pin;
 use serde::{Deserialize, Serialize};
@@ -107,6 +107,27 @@ macro_rules! static_http {
     };
 }
 
+macro_rules! static_http_api1 {
+    ($path:expr, $tgt:expr, $tgtex:expr, $ctype:expr) => {
+        if $path == concat!("/api/1/documentation/", $tgt) {
+            let c = include_bytes!(concat!("../static/documentation/", $tgtex));
+            let ret = response(StatusCode::OK)
+                .header("content-type", $ctype)
+                .body(Body::from(&c[..]))?;
+            return Ok(ret);
+        }
+    };
+    ($path:expr, $tgt:expr, $ctype:expr) => {
+        if $path == concat!("/api/1/documentation/", $tgt) {
+            let c = include_bytes!(concat!("../static/documentation/", $tgt));
+            let ret = response(StatusCode::OK)
+                .header("content-type", $ctype)
+                .body(Body::from(&c[..]))?;
+            return Ok(ret);
+        }
+    };
+}
+
 async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
     let uri = req.uri().clone();
     let path = uri.path();
@@ -177,9 +198,24 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
         } else {
             Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
         }
+    } else if path == "/api/4/channel/config" {
+        if req.method() == Method::GET {
+            Ok(channel_config(req, &node_config).await?)
+        } else {
+            Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
+        }
+    } else if path.starts_with("/api/1/documentation/") {
+        if req.method() == Method::GET {
+            static_http_api1!(path, "", "api1.html", "text/html");
+            static_http_api1!(path, "style.css", "text/css");
+            static_http_api1!(path, "script.js", "text/javascript");
+            Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
+        } else {
+            Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
+        }
     } else if path.starts_with("/api/4/documentation/") {
         if req.method() == Method::GET {
-            static_http!(path, "", "index.html", "text/html");
+            static_http!(path, "", "api4.html", "text/html");
             static_http!(path, "style.css", "text/css");
             static_http!(path, "script.js", "text/javascript");
             static_http!(path, "status-main.html", "text/html");
@@ -446,3 +482,21 @@ pub async fn update_search_cache(req: Request<Body>, node_config: &NodeConfigCac
         .body(Body::from(serde_json::to_string(&res)?))?;
     Ok(ret)
 }
+
+pub async fn channel_config(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
+    let (head, _body) = req.into_parts();
+    let _dry = match head.uri.query() {
+        Some(q) => q.contains("dry"),
+        None => false,
+    };
+    let params = netpod::query_params(head.uri.query());
+    let channel = Channel {
+        backend: node_config.node.backend.clone(),
+        name: params.get("channelName").unwrap().into(),
+    };
+    let res = parse::channelconfig::read_local_config(&channel, &node_config.node).await?;
+    let ret = response(StatusCode::OK)
+        .header(http::header::CONTENT_TYPE, "application/json")
+        .body(Body::from(serde_json::to_string(&res)?))?;
+    Ok(ret)
+}

httpret/static/documentation/api1.html

@@ -0,0 +1,159 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8"/>
+    <title>Databuffer API 1 Documentation</title>
+    <meta name="keywords" content="PSI, DAQ, Databuffer">
+    <meta name="author" content="Dominik Werder">
+    <link rel="shortcut icon" href="about:blank"/>
+    <link rel="stylesheet" href="style.css"/>
+</head>
+
+<body>
+
+<h1>Databuffer API 1 Documentation</h1>
+
+<h2>Available backends</h2>
+Currently available backends:
+<ul>
+    <li>sf-databuffer</li>
+    <li>sf-imagebuffer</li>
+    <li>hipa-archive</li>
+    <li>gls-archive</li>
+</ul>
+
+<h2>API functions</h2>
+<p>Currently available:</p>
+<ul>
+    <li><a href="#channel-search-names">Channel search, with return of channel names</a></li>
+    <li><a href="#channel-search-configs">Channel search, with return of channel configurations</a></li>
+</ul>
+
+
+<a id="channel-search-names"></a>
+<h2>Channel Search, returns only channel names</h2>
+<p><strong>Method:</strong> POST</p>
+<p><strong>URL:</strong> https://data-api.psi.ch/api/1/channels</p>
+<p><strong>Request body:</strong> JSON with search parameters</p>
+<p><strong>Request body outline:</strong></p>
+<pre>
+{
+  "regex":             "[Optional: Regular expression to search in channel name]",
+  "sourceRegex":       "[Optional: Search in sourcename of the channel]",
+  "descriptionRegex":  "[Optional: Search in the channel's description]",
+  "backends":          ["gls-archive", "hipa-archive", "sf-databuffer"]
+}
+</pre>
+<p><strong>Request body example:</strong></p>
+<pre>
+{
+  "regex": "SARES20-LSCP9:CH0",
+  "backends": ["sf-databuffer", "hipa-archive"]
+}
+</pre>
+<p><strong>Result body example:</strong></p>
+<p>Assuming that "hipa-archive" would be unavailable:</p>
+<pre>
+[
+    {
+        "backend": "sf-databuffer",
+        "channels": [
+            "SARES20-LSCP9:CH0:2",
+            "SARES20-LSCP9:CH0:1"
+        ]
+    },
+    {
+        "backend": "hipa-archive",
+        "channels": [],
+        "error": {
+            "code": "Error"  // can be: "Error" | "Timeout" (more to be added in the future)
+        }
+    }
+]
+</pre>
+<p>Notes:</p>
+<p>The search constraints are AND'ed together.</p>
+<p>If some backend responds with an error, that error is indicated by the error key in the affected backend (see example above).</p>
+
+
+<h4>CURL example:</h4>
+<pre>
+QUERY='{ "regex": "LSCP9:CH0", "backends": ["sf-databuffer"] }'
+curl -H 'Content-Type: application/json' -H 'Accept: application/json' -d "$QUERY" https://data-api.psi.ch/api/1/channels
+</pre>
+
+
+<a id="channel-search-configs"></a>
+<h2>Channel Search, with return of configuration information</h2>
+<p><strong>Method:</strong> POST</p>
+<p><strong>URL:</strong> https://data-api.psi.ch/api/1/channels/config</p>
+<p><strong>Request body:</strong> JSON with search parameters</p>
+<p><strong>Request body outline:</strong></p>
+<pre>
+{
+  "regex":             "[Optional: Regular expression to search in channel name]",
+  "sourceRegex":       "[Optional: Search in sourcename of the channel]",
+  "descriptionRegex":  "[Optional: Search in the channel's description]",
+  "backends":          ["gls-archive", "hipa-archive", "sf-databuffer"]
+}
+</pre>
+<p><strong>Result body example:</strong></p>
+<p>Assuming that "hipa-archive" would be unavailable:</p>
+<pre>
+[
+    {
+        "backend": "sf-databuffer",
+        "channels": [
+            {
+                "backend": "sf-databuffer",
+                "description": "",
+                "name": "SARES20-LSCP9:CH0:2",
+                "shape": [
+                    512
+                ],
+                "source": "tcp://SARES20-CVME-01:9999",
+                "type": "Float32",
+                "unit": ""
+            },
+            {
+                "backend": "sf-databuffer",
+                "description": "",
+                "name": "SARES20-LSCP9:CH0:1",
+                "shape": [
+                    512
+                ],
+                "source": "tcp://SARES20-CVME-01:9999",
+                "type": "Int16",
+                "unit": ""
+            }
+        ]
+    },
+    {
+        "backend": "hipa-archive",
+        "channels": [],
+        "error": {
+            "code": "Error"  // can be: "Error" | "Timeout" (more to be added in the future)
+        }
+    }
+]
+</pre>
+<p>Notes:</p>
+<p>The search constraints are AND'ed together.</p>
+<p>If some backend responds with an error, that error is indicated by the error key in the affected backend (see example above).</p>
+
+
+<h4>CURL example:</h4>
+<pre>
+QUERY='{ "regex": "LSCP9:CH0", "backends": ["sf-databuffer"] }'
+curl -H 'Content-Type: application/json' -H 'Accept: application/json' -d "$QUERY" https://data-api.psi.ch/api/1/channels/config
+</pre>
+
+<h2>Feedback and comments</h2>
+<p>Feedback is very much appreciated:</p>
+<p>dominik.werder@psi.ch</p>
+<p>or please assign me a JIRA ticket.</p>
+
+<div id="footer"></div>
+
+</body>
+</html>

httpret/static/documentation/api4.html

@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
     <meta charset="utf-8"/>
-    <title>Retrieval Documentation</title>
+    <title>Databuffer API 4 Documentation</title>
     <meta name="keywords" content="PSI, DAQ, Databuffer">
     <meta name="author" content="Dominik Werder">
     <link rel="shortcut icon" href="about:blank"/>
@@ -11,13 +11,19 @@
 
 <body>
 
-<h1>Retrieval /api/4 Documentation</h1>
+<h1>Databuffer API 4 Documentation</h1>
 
-<h2>HTTP API documentation</h2>
+<p>Documented here are the endpoints for databuffer API 4. The endpoints of the "original" unversioned API is documented at
+<a href="https://git.psi.ch/sf_daq/ch.psi.daq.databuffer/blob/master/ch.psi.daq.queryrest/Readme.md">this location</a>.</p>
 
-<h3>Limitations</h3>
+<h2>Available backends</h2>
-<p>These services currently only serve data from the sf-databuffer backend.</p>
+Currently available:
+<ul>
+    <li>sf-databuffer</li>
+</ul>
 
+
+<h2>API functions</h2>
 <p>Currently available functionality:</p>
 <ul>
     <li><a href="#query-binned">Query binned data</a></li>
@@ -140,6 +146,7 @@ curl -H 'Accept: application/json' 'https://data-api.psi.ch/api/4/search/channel
   ]
 }
 </pre>
+<p>The search constraints are AND'd.</p>
 
 
 <h2>Feedback and comments very much appreciated!</h2>