Move API 1 docs here

2021-06-02 07:24:00 +02:00
parent 3d17fea919
commit 53cf26d830
6 changed files with 241 additions and 40 deletions
--- a/dbconn/src/lib.rs
+++ b/dbconn/src/lib.rs
@@ -33,20 +33,6 @@ pub async fn create_connection(node_config: &NodeConfigCached) -> Result<Client,
    Ok(cl)
 }

-pub async fn create_connection_2(node_config: NodeConfigCached) -> Result<Client, Error> {
-    let d = &node_config.node_config.cluster.database;
-    let uri = format!("postgresql://{}:{}@{}:{}/{}", d.user, d.pass, d.host, 5432, d.name);
-    let (cl, conn) = tokio_postgres::connect(&uri, NoTls).await?;
-    // TODO monitor connection drop.
-    let _cjh = tokio::spawn(async move {
-        if let Err(e) = conn.await {
-            error!("connection error: {}", e);
-        }
-        Ok::<_, Error>(())
-    });
-    Ok(cl)
-}
-
 pub async fn channel_exists(channel: &Channel, node_config: &NodeConfigCached) -> Result<bool, Error> {
    let cl = create_connection(node_config).await?;
    let rows = cl
--- a/dbconn/src/scan.rs
+++ b/dbconn/src/scan.rs
@@ -1,4 +1,4 @@
-use async_channel::{bounded, Receiver, Sender};
+use async_channel::{bounded, Receiver};
 use chrono::{DateTime, Utc};
 use err::Error;
 use futures_core::Stream;
@@ -8,11 +8,9 @@ use netpod::NodeConfigCached;
 use serde::{Deserialize, Serialize};
 use std::future::Future;
 use std::io::ErrorKind;
-use std::marker::PhantomPinned;
 use std::os::unix::ffi::OsStringExt;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
-use std::ptr::NonNull;
 use std::sync::{Arc, RwLock};
 use std::task::{Context, Poll};
 use tokio_postgres::Client;
@@ -80,12 +78,12 @@ pub struct UpdatedDbWithChannelNames {
 pub struct UpdatedDbWithChannelNamesStream {
    errored: bool,
    data_complete: bool,
+    #[allow(dead_code)]
    node_config: Pin<Box<NodeConfigCached>>,
-    node_config_ptr: NonNull<NodeConfigCached>,
-    //_pin: PhantomPinned,
+    node_config_ref: &'static NodeConfigCached,
    client_fut: Option<Pin<Box<dyn Future<Output = Result<Client, Error>> + Send>>>,
    client: Option<Pin<Box<Client>>>,
-    client_ptr: *const Client,
+    client_ref: Option<&'static Client>,
    ident_fut: Option<Pin<Box<dyn Future<Output = Result<NodeDiskIdent, Error>> + Send>>>,
    ident: Option<NodeDiskIdent>,
 }
@@ -95,22 +93,19 @@ unsafe impl Send for UpdatedDbWithChannelNamesStream {}
 impl UpdatedDbWithChannelNamesStream {
    pub fn new(node_config: NodeConfigCached) -> Result<Self, Error> {
        let node_config = Box::pin(node_config.clone());
+        let node_config_ref = unsafe { &*(&node_config as &NodeConfigCached as *const _) };
        let mut ret = Self {
            errored: false,
            data_complete: false,
-            node_config_ptr: NonNull::dangling(),
            node_config,
-            //_pin: PhantomPinned,
+            node_config_ref,
            client_fut: None,
            client: None,
-            client_ptr: std::ptr::null(),
+            client_ref: None,
            ident_fut: None,
            ident: None,
        };
-        ret.node_config_ptr = NonNull::from(&*ret.node_config);
-        ret.client_fut = Some(Box::pin(crate::create_connection(unsafe {
-            &*ret.node_config_ptr.as_ptr()
-        })));
+        ret.client_fut = Some(Box::pin(crate::create_connection(ret.node_config_ref)));
        Ok(ret)
    }
 }
@@ -120,7 +115,7 @@ impl Stream for UpdatedDbWithChannelNamesStream {

    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
        use Poll::*;
-        'outer: loop {
+        loop {
            break if self.errored {
                Ready(None)
            } else if self.data_complete {
@@ -148,11 +143,10 @@ impl Stream for UpdatedDbWithChannelNamesStream {
                    Ready(Ok(item)) => {
                        self.client_fut = None;
                        self.client = Some(Box::pin(item));
-                        self.client_ptr = self.client.as_ref().unwrap() as &Client as *const _;
-                        let p1 = unsafe { &*self.client_ptr };
+                        self.client_ref = Some(unsafe { &*(&self.client.as_ref().unwrap() as &Client as *const _) });
                        self.ident_fut = Some(Box::pin(get_node_disk_ident(
-                            unsafe { &*self.node_config_ptr.as_ptr() },
-                            &p1,
+                            self.node_config_ref,
+                            self.client_ref.as_ref().unwrap(),
                        )));
                        let ret = UpdatedDbWithChannelNames {
                            msg: format!("Client opened connection"),
@@ -209,7 +203,7 @@ pub async fn update_db_with_channel_names(
    })
    .await?;
    dbc.query("commit", &[]).await?;
-    let ret = UpdatedDbWithChannelNames {
+    let _ret = UpdatedDbWithChannelNames {
        msg: format!("done"),
        count: *c1.read()?,
    };
--- a/httpret/Cargo.toml
+++ b/httpret/Cargo.toml
@@ -20,4 +20,5 @@ err = { path = "../err" }
 netpod = { path = "../netpod" }
 dbconn = { path = "../dbconn" }
 disk = { path = "../disk" }
+parse = { path = "../parse" }
 taskrun = { path = "../taskrun" }
--- a/httpret/src/lib.rs
+++ b/httpret/src/lib.rs
@@ -12,7 +12,7 @@ use hyper::service::{make_service_fn, service_fn};
 use hyper::{server::Server, Body, Request, Response};
 use net::SocketAddr;
 use netpod::log::*;
-use netpod::NodeConfigCached;
+use netpod::{Channel, NodeConfigCached};
 use panic::{AssertUnwindSafe, UnwindSafe};
 use pin::Pin;
 use serde::{Deserialize, Serialize};
@@ -107,6 +107,27 @@ macro_rules! static_http {
    };
 }

+macro_rules! static_http_api1 {
+    ($path:expr, $tgt:expr, $tgtex:expr, $ctype:expr) => {
+        if $path == concat!("/api/1/documentation/", $tgt) {
+            let c = include_bytes!(concat!("../static/documentation/", $tgtex));
+            let ret = response(StatusCode::OK)
+                .header("content-type", $ctype)
+                .body(Body::from(&c[..]))?;
+            return Ok(ret);
+        }
+    };
+    ($path:expr, $tgt:expr, $ctype:expr) => {
+        if $path == concat!("/api/1/documentation/", $tgt) {
+            let c = include_bytes!(concat!("../static/documentation/", $tgt));
+            let ret = response(StatusCode::OK)
+                .header("content-type", $ctype)
+                .body(Body::from(&c[..]))?;
+            return Ok(ret);
+        }
+    };
+}
+
 async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
    let uri = req.uri().clone();
    let path = uri.path();
@@ -177,9 +198,24 @@ async fn http_service_try(req: Request<Body>, node_config: &NodeConfigCached) ->
        } else {
            Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
        }
+    } else if path == "/api/4/channel/config" {
+        if req.method() == Method::GET {
+            Ok(channel_config(req, &node_config).await?)
+        } else {
+            Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
+        }
+    } else if path.starts_with("/api/1/documentation/") {
+        if req.method() == Method::GET {
+            static_http_api1!(path, "", "api1.html", "text/html");
+            static_http_api1!(path, "style.css", "text/css");
+            static_http_api1!(path, "script.js", "text/javascript");
+            Ok(response(StatusCode::NOT_FOUND).body(Body::empty())?)
+        } else {
+            Ok(response(StatusCode::METHOD_NOT_ALLOWED).body(Body::empty())?)
+        }
    } else if path.starts_with("/api/4/documentation/") {
        if req.method() == Method::GET {
-            static_http!(path, "", "index.html", "text/html");
+            static_http!(path, "", "api4.html", "text/html");
            static_http!(path, "style.css", "text/css");
            static_http!(path, "script.js", "text/javascript");
            static_http!(path, "status-main.html", "text/html");
@@ -446,3 +482,21 @@ pub async fn update_search_cache(req: Request<Body>, node_config: &NodeConfigCac
        .body(Body::from(serde_json::to_string(&res)?))?;
    Ok(ret)
 }
+
+pub async fn channel_config(req: Request<Body>, node_config: &NodeConfigCached) -> Result<Response<Body>, Error> {
+    let (head, _body) = req.into_parts();
+    let _dry = match head.uri.query() {
+        Some(q) => q.contains("dry"),
+        None => false,
+    };
+    let params = netpod::query_params(head.uri.query());
+    let channel = Channel {
+        backend: node_config.node.backend.clone(),
+        name: params.get("channelName").unwrap().into(),
+    };
+    let res = parse::channelconfig::read_local_config(&channel, &node_config.node).await?;
+    let ret = response(StatusCode::OK)
+        .header(http::header::CONTENT_TYPE, "application/json")
+        .body(Body::from(serde_json::to_string(&res)?))?;
+    Ok(ret)
+}
--- a/httpret/static/documentation/api1.html
+++ b/httpret/static/documentation/api1.html
@@ -0,0 +1,159 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8"/>
+    <title>Databuffer API 1 Documentation</title>
+    <meta name="keywords" content="PSI, DAQ, Databuffer">
+    <meta name="author" content="Dominik Werder">
+    <link rel="shortcut icon" href="about:blank"/>
+    <link rel="stylesheet" href="style.css"/>
+</head>
+
+<body>
+
+<h1>Databuffer API 1 Documentation</h1>
+
+<h2>Available backends</h2>
+Currently available backends:
+<ul>
+    <li>sf-databuffer</li>
+    <li>sf-imagebuffer</li>
+    <li>hipa-archive</li>
+    <li>gls-archive</li>
+</ul>
+
+<h2>API functions</h2>
+<p>Currently available:</p>
+<ul>
+    <li><a href="#channel-search-names">Channel search, with return of channel names</a></li>
+    <li><a href="#channel-search-configs">Channel search, with return of channel configurations</a></li>
+</ul>
+
+
+<a id="channel-search-names"></a>
+<h2>Channel Search, returns only channel names</h2>
+<p><strong>Method:</strong> POST</p>
+<p><strong>URL:</strong> https://data-api.psi.ch/api/1/channels</p>
+<p><strong>Request body:</strong> JSON with search parameters</p>
+<p><strong>Request body outline:</strong></p>
+<pre>
+{
+  "regex":             "[Optional: Regular expression to search in channel name]",
+  "sourceRegex":       "[Optional: Search in sourcename of the channel]",
+  "descriptionRegex":  "[Optional: Search in the channel's description]",
+  "backends":          ["gls-archive", "hipa-archive", "sf-databuffer"]
+}
+</pre>
+<p><strong>Request body example:</strong></p>
+<pre>
+{
+  "regex": "SARES20-LSCP9:CH0",
+  "backends": ["sf-databuffer", "hipa-archive"]
+}
+</pre>
+<p><strong>Result body example:</strong></p>
+<p>Assuming that "hipa-archive" would be unavailable:</p>
+<pre>
+[
+    {
+        "backend": "sf-databuffer",
+        "channels": [
+            "SARES20-LSCP9:CH0:2",
+            "SARES20-LSCP9:CH0:1"
+        ]
+    },
+    {
+        "backend": "hipa-archive",
+        "channels": [],
+        "error": {
+            "code": "Error"  // can be: "Error" | "Timeout" (more to be added in the future)
+        }
+    }
+]
+</pre>
+<p>Notes:</p>
+<p>The search constraints are AND'ed together.</p>
+<p>If some backend responds with an error, that error is indicated by the error key in the affected backend (see example above).</p>
+
+
+<h4>CURL example:</h4>
+<pre>
+QUERY='{ "regex": "LSCP9:CH0", "backends": ["sf-databuffer"] }'
+curl -H 'Content-Type: application/json' -H 'Accept: application/json' -d "$QUERY" https://data-api.psi.ch/api/1/channels
+</pre>
+
+
+<a id="channel-search-configs"></a>
+<h2>Channel Search, with return of configuration information</h2>
+<p><strong>Method:</strong> POST</p>
+<p><strong>URL:</strong> https://data-api.psi.ch/api/1/channels/config</p>
+<p><strong>Request body:</strong> JSON with search parameters</p>
+<p><strong>Request body outline:</strong></p>
+<pre>
+{
+  "regex":             "[Optional: Regular expression to search in channel name]",
+  "sourceRegex":       "[Optional: Search in sourcename of the channel]",
+  "descriptionRegex":  "[Optional: Search in the channel's description]",
+  "backends":          ["gls-archive", "hipa-archive", "sf-databuffer"]
+}
+</pre>
+<p><strong>Result body example:</strong></p>
+<p>Assuming that "hipa-archive" would be unavailable:</p>
+<pre>
+[
+    {
+        "backend": "sf-databuffer",
+        "channels": [
+            {
+                "backend": "sf-databuffer",
+                "description": "",
+                "name": "SARES20-LSCP9:CH0:2",
+                "shape": [
+                    512
+                ],
+                "source": "tcp://SARES20-CVME-01:9999",
+                "type": "Float32",
+                "unit": ""
+            },
+            {
+                "backend": "sf-databuffer",
+                "description": "",
+                "name": "SARES20-LSCP9:CH0:1",
+                "shape": [
+                    512
+                ],
+                "source": "tcp://SARES20-CVME-01:9999",
+                "type": "Int16",
+                "unit": ""
+            }
+        ]
+    },
+    {
+        "backend": "hipa-archive",
+        "channels": [],
+        "error": {
+            "code": "Error"  // can be: "Error" | "Timeout" (more to be added in the future)
+        }
+    }
+]
+</pre>
+<p>Notes:</p>
+<p>The search constraints are AND'ed together.</p>
+<p>If some backend responds with an error, that error is indicated by the error key in the affected backend (see example above).</p>
+
+
+<h4>CURL example:</h4>
+<pre>
+QUERY='{ "regex": "LSCP9:CH0", "backends": ["sf-databuffer"] }'
+curl -H 'Content-Type: application/json' -H 'Accept: application/json' -d "$QUERY" https://data-api.psi.ch/api/1/channels/config
+</pre>
+
+<h2>Feedback and comments</h2>
+<p>Feedback is very much appreciated:</p>
+<p>dominik.werder@psi.ch</p>
+<p>or please assign me a JIRA ticket.</p>
+
+<div id="footer"></div>
+
+</body>
+</html>
--- a/httpret/static/documentation/index.html
+++ b/httpret/static/documentation/index.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
    <meta charset="utf-8"/>
-    <title>Retrieval Documentation</title>
+    <title>Databuffer API 4 Documentation</title>
    <meta name="keywords" content="PSI, DAQ, Databuffer">
    <meta name="author" content="Dominik Werder">
    <link rel="shortcut icon" href="about:blank"/>
@@ -11,13 +11,19 @@

 <body>

-<h1>Retrieval /api/4 Documentation</h1>
+<h1>Databuffer API 4 Documentation</h1>

-<h2>HTTP API documentation</h2>
+<p>Documented here are the endpoints for databuffer API 4. The endpoints of the "original" unversioned API is documented at
+<a href="https://git.psi.ch/sf_daq/ch.psi.daq.databuffer/blob/master/ch.psi.daq.queryrest/Readme.md">this location</a>.</p>

-<h3>Limitations</h3>
-<p>These services currently only serve data from the sf-databuffer backend.</p>
+<h2>Available backends</h2>
+Currently available:
+<ul>
+    <li>sf-databuffer</li>
+</ul>

+
+<h2>API functions</h2>
 <p>Currently available functionality:</p>
 <ul>
    <li><a href="#query-binned">Query binned data</a></li>
@@ -140,6 +146,7 @@ curl -H 'Accept: application/json' 'https://data-api.psi.ch/api/4/search/channel
  ]
 }
 </pre>
+<p>The search constraints are AND'd.</p>


 <h2>Feedback and comments very much appreciated!</h2>