major rework

using influxdb structure independed of nicos cache

base.py

@@ -6,9 +6,11 @@ ONEYEAR = 366 * 24 * 3600
 
 
 def get_abs_time(times):
-    """
-    Gets the absolute times for the given potential relative times. If a given timestamp is less than
-    one year, then the value is relative and converted into an absolute timestamp
+    """Gets the absolute times for the given potential relative times.
+
+    If a given timestamp is less than one year, then the value is
+    relative (to now, rounded up to a full second) and converted
+    into an absolute timestamp
 
     Parameters :
         times([(float)]) : an array of unix timestamps or relative duration (< 1 year) as floats

chart_config.py

@@ -1,4 +1,6 @@
 from configparser import ConfigParser
+
+
 class ChartConfig:
     """
     Class that holds the chart section of a configuration file (for an instrument).
@@ -6,14 +8,42 @@ class ChartConfig:
     Attributes :
         chart_config (Section) : the Section corresponding to the "chart" section in the given configuration file
     """
+
+    KEYS = ["cat", "color", "unit", "label"]
+
     def __init__(self, path):
         """
         Parameters :
             path (str) : the path to the configuration file
         """
+        self.errors = {}
+        self.variables = {}
         cfgp = ConfigParser(interpolation=None)
+        cfgp.optionxform = str
         cfgp.read(path)
-        self.chart_config = cfgp["chart"]
+        section = cfgp["chart"]
+        for key, raw_value in section.items():
+            if len(key.split('.')) > 1:
+                self.errors[key] = f'illegal key: {key}'
+                continue
+            arguments = raw_value.split(",")
+            keyword_mode = False
+            config = {'cat': '*'}
+            for i, argument in enumerate(arguments):
+                argname, _, argvalue = argument.rpartition(':')
+                if argname:
+                    keyword_mode = True
+                    config[argname] = argvalue
+                else:
+                    if keyword_mode:
+                        self.errors[key] = f"positional arg after keywd arg: {key}={raw_value!r}"
+                    else:
+                        try:
+                            if argvalue:
+                                config[self.KEYS[i]] = argvalue
+                        except Exception as e:
+                            self.errors[key] = f"{e!r} in {key}={raw_value}"
+            self.variables[key] = config
 
     def get_variable_parameter_config(self, key):
         """
@@ -27,25 +57,4 @@ class ChartConfig:
                 The different options are in this dict if they are found in the chart section for the given key. Returns None if the key is not in the chart section,
                 or if there is a syntax problem for the given key.
         """
-        config = {}
-        positionnal = ["cat", "color", "unit"]
-        if key in self.chart_config.keys():
-            raw_value = self.chart_config[key]
-
-            arguments = raw_value.split(",")
-            keyword_mode = False
-            for i, argument in enumerate(arguments):
-                pieces = argument.split(":")
-                if len(pieces) == 2:
-                    keyword_mode = True
-                    if pieces[1] != "":
-                        config[pieces[0]] = pieces[1]
-                else:
-                    if not keyword_mode: #everything is going well
-                        if pieces[0] != "":
-                            config[positionnal[i]] = pieces[0]
-                    else: #we cannot have a positionnal argument after a keyword argument
-                        return None
-            return config
-        else:
-            return None
+        return self.variables.get(key)

client/jsFiles/SEAWebClientCommunication.js

@@ -111,8 +111,8 @@ function handleUpdateMessage(src, message) {
         instrument.style.width = 'auto'
         device.style.width = 'auto'
         instrument.innerHTML = message.instrument
-        device.innerHTML = message.device
-        // console.log('ID', initCommands);
+        // device.innerHTML = message.device
+        console.log('ID', initCommands);
         nextInitCommand();
         break;
     // console-update-message: Confirms a command.
@@ -306,7 +306,7 @@ function reqJSONPOST(s, url, parameters, successHandler, errorHandler) {
     var xhr = typeof XMLHttpRequest != 'undefined' ? new XMLHttpRequest()
             : new ActiveXObject('Microsoft.XMLHTTP');
     if (debugCommunication) {
-        console.log("%cto server (reqJSON): %s",
+        console.log("%cto server (reqJSONPOST): %s",
                 "color:white;background:lightgreen", url);
     }
     xhr.open('post', url, true);
@@ -406,8 +406,11 @@ function successHandler(s, message) {
         begin = timeRange[0] - timeRange[1];
         select.value = begin;
         // Server-request for variable-list.*/
-        reqJSONPOST(0, "http://" + hostPort + "/getvars", "time=" + timeRange[1] + "&userconfiguration=" + JSON.stringify(getFormattedUserConfigurationFromLocalStorage()) + "&id=" 
-            + clientID, successHandler, errorHandler);
+        reqJSONPOST(0, "http://" + hostPort + "/getvars",
+                "time=" + timeRange[1]
+                + window['clientTags']
+                + "&userconfiguration=" + JSON.stringify(getFormattedUserConfigurationFromLocalStorage())
+                + "&id=" + clientID, successHandler, errorHandler);
         break;
     // Response to a "getvars"-server-request.
     case "var_list":
@@ -427,6 +430,7 @@ function successHandler(s, message) {
             nextInitCommand();
         }*/
         // graphs.receivedVars(message.blocks);
+        document.getElementById("device").innerHTML = message.device
         graphs.initGraphs(message.blocks);
         nextInitCommand();
         break;

client/jsFiles/SEAWebClientGraphics.js

@@ -371,10 +371,9 @@ function loadExportPopup(){
  */
 function exportCallback(selectedVariables, startDateTimeMs, endDateTimeMs, nan, binning=null){
 
-    let binningParam = "None";
-    if (binning !== null)
-        binningParam = binning
-    let exportURL = "http://" + hostPort + "/export?time=" + startDateTimeMs/1000 + "," + endDateTimeMs/1000 + "&variables=" + selectedVariables + "&nan=" + nan + "&interval=" + binningParam + "&id=" + clientID
+    if (binning === null || binning == "None")
+        binning = "";
+    let exportURL = "http://" + hostPort + "/export?time=" + startDateTimeMs/1000 + "," + endDateTimeMs/1000 + "&variables=" + selectedVariables + "&nan=" + nan + "&interval=" + binning + "&id=" + clientID
     let a = document.createElement('a');
     a.href = exportURL
     a.download = true
@@ -417,7 +416,7 @@ let graphs = (function (){
     let minTime, maxTime; // the queried time range
     let lastTime = 0; // time of most recent data point
 
-    let resolution = undefined; // current window resolution (ms/pixel)
+    // let resolution = undefined; // current window resolution (ms/pixel)
 
     let activateUpdateTimeout = undefined; // timeout for the activateUpdates function
     let updateAutoTimeout = undefined; // timeout for the updateAuto function (used in onZoomCompleteCallback)
@@ -542,11 +541,12 @@ let graphs = (function (){
         varlist = vars_array[gindex];
         let graph_elm = graph_elm_array[gindex];
 
-        timeDeltaAxis = maxTime - minTime
-        setResolution(timeDeltaAxis)
-
-        AJAX("http://" + hostPort + "/graph?time=" + minTime/1000 + "," + maxTime/1000 + "&variables=" + varlist + "&interval=" + resolution + "&id=" + clientID).getJSON().then(function(data){
+        resolution = getResolution((maxTime - minTime) / 1000)
 
+        AJAX("http://" + hostPort + "/graph?time=" + minTime/1000 + "," + maxTime/1000
+                + "&variables=" + varlist
+                + "&interval=" + resolution
+                + "&id=" + clientID).getJSON().then(function(data){
             //console.log('Graph', block, data)
             let graph = new Graph(gindex, graph_elm, "Time", block.unit, block.tag, type);
             graph_array[gindex] = graph;
@@ -777,10 +777,12 @@ let graphs = (function (){
             max = max/1000;
         }
         
-        timeDelta = currentMaxTime - currentMinTime
-        setResolution(timeDelta)
+        resolution = getResolution((currentMaxTime - currentMinTime) / 1000)
 
-        AJAX("http://" + hostPort + "/graph?time=" + min + ","+max+"&variables=" + variables() + "&interval=" + resolution +  "&id=" + clientID).getJSON().then(function(data){
+        AJAX("http://" + hostPort + "/graph?time=" + min + ","+max
+                +"&variables=" + variables()
+                + "&interval=" + resolution
+                + "&id=" + clientID).getJSON().then(function(data){
             for(let key in data.graph){
                 let pdata = [];
                 for(let e of data.graph[key]){
@@ -893,10 +895,10 @@ let graphs = (function (){
      * Sets the resolution of the viewing window in milliseconds
      * @param {*} timeDelta - The difference between the maximum time and the minimum time of the window
      */
-    function setResolution(timeDelta){
-        resolution = Math.ceil((timeDelta / container.getBoundingClientRect().width))
+    function getResolution(timeDelta){
+        return Math.ceil((timeDelta / container.getBoundingClientRect().width))
     }
-    
+
     /**
      * The callback to be called when the user click on the "Jump" button of the date selector
      * Gets the vars + device name for the selected date+time, then rebuilds the graphs
@@ -915,7 +917,11 @@ let graphs = (function (){
         msRightTimestampGetVars = dateTimestampMs + timeValueMs;
         msRightTimestampGetGraph = dateTimestampMs + 24*60*60*1000;
 
-        AJAX("http://" + hostPort + "/getvars").postForm("time=" + msRightTimestampGetVars/1000 + "&userconfiguration=" + JSON.stringify(getFormattedUserConfigurationFromLocalStorage()) + "&id="+ clientID).then(function(data){
+        AJAX("http://" + hostPort + "/getvars").postForm(
+                "time=" + msRightTimestampGetVars/1000
+                + window['clientTags']
+                + "&userconfiguration=" + JSON.stringify(getFormattedUserConfigurationFromLocalStorage())
+                + "&id="+ clientID).then(function(data){
             blocks = data.blocks;
             document.getElementById("device").innerHTML =  data.device
             maxTime = msRightTimestampGetGraph;
@@ -989,7 +995,11 @@ let graphs = (function (){
         window["wideGraphs"] = false; // will have no effect if hideRightPart is true
         adjustGrid();
 
-        AJAX("http://" + hostPort + "/getvars").postForm("time=" + msRightTimestamp/1000 + "&userconfiguration=" + JSON.stringify(getFormattedUserConfigurationFromLocalStorage()) + "&id="+ clientID).then(function(data){
+        AJAX("http://" + hostPort + "/getvars").postForm(
+                "time=" + msRightTimestamp/1000 + "&userconfiguration="
+                + JSON.stringify(getFormattedUserConfigurationFromLocalStorage())
+                + window['clientTags']
+                + "&id="+ clientID).then(function(data){
             currentMaxTime = msRightTimestamp + 60000;
             currentMinTime = msLeftTimestamp;
 
@@ -1291,7 +1301,11 @@ let graphs = (function (){
     function applySettingsCallback(userConfiguration){
         cursorLine(null);
 
-        AJAX("http://" + hostPort + "/getvars").postForm("time=" + currentMaxTime/1000 + "&userconfiguration=" + JSON.stringify(userConfiguration) + "&id="+ clientID).then(function(data){
+        AJAX("http://" + hostPort + "/getvars").postForm(
+                "time=" + currentMaxTime/1000
+                + "&userconfiguration=" + JSON.stringify(userConfiguration)
+                + window['clientTags']
+                + "&id="+ clientID).then(function(data){
             blocks = data.blocks;
             document.getElementById("device").innerHTML =  data.device
             maxTime = currentMaxTime;

client/jsFiles/SEAWebClientMain.js

@@ -91,6 +91,17 @@ new Settings()
     .treat("hideRightPart", "hr", to_bool, false) //used to completely disable the right part
     .treat("wideGraphs", "wg", to_bool, false) //used to toggle the size of the graphs part
     .treat("showAsync", "sa", to_bool, false)
+    .treat("device", "dev", 0, "*")
+    .treat("server", "srv", 0, "*")
+    .treat("instrument", "ins", 0, "")
+
+if (window['instrument']) {
+     window['clientTags'] = "&instrument=" + window["instrument"];
+} else {
+     window['clientTags'] = "&stream=" + window["server"] + "&device=" + window["device"];
+}
+
+console.log('TAGS', window['clientTags']);
 
 function loadFirstBlocks() {
     if (debug_main_daniel) {
@@ -196,9 +207,9 @@ window.onload = function() {
     // var homeButton = document.getElementById("home-icon");
     
     // TODO : uncomment this code with the right URL to navigate to when the way to select the instrument will be decided.
-    // homeButton.onclick = function () {
-    //     window.location = "http://" + location.hostname + ":8800/";
-    // };
+    homeButton.onclick = function () {
+        window.location = "/select";
+    };
     buildUpdateConnection();
   //   if (location.hash) {
   //     console.log("hash in url", location.hash);

colors.py

@@ -18,7 +18,7 @@ def assign_colors_to_curves(blocks):
         auto_curves = []
         for curve in block["curves"]:
 
-            col = curve["color"].strip()
+            col = curve.get("color", "").strip()
             c = ColorMap.to_code(col)
             if c < 0:
                 valid = ColorMap.check_hex(col)

config/generic.ini

@@ -20,4 +20,15 @@ T_sorb.target=-
 T_still=unit:K,color:orange
 dil=-
 lev=unit:%,color:brown
-lev.n2=*,unit:%,color:black
+lev.n2=unit:%,color:black
+hefill=-
+ln2fill=-
+hepump=-
+hemot=-
+nv=-
+nv.flow=unit:ln/min
+nv.flowtarget=unit:ln/min
+nv.flowp=unit:ln/min
+stickrot=unit:deg
+tcoil1=*_coil,unit:K
+tcoil2=*_coil,unit:K

config/influx.ini

@@ -1,4 +1,5 @@
 [INFLUX]
-url=http://localhost:8086
+url=http://linse-a:8086
 org=linse
+bucket=curve-test
 token=zqDbTcMv9UizfdTj15Fx_6vBetkM5mXN56EE9CiDaFsh7O2FFWZ2X4VwAAmdyqZr3HbpIr5ixRju07-oQmxpXw==

influxdb.py

@@ -17,6 +17,7 @@ class InfluxDB:
 
     def __init__(self):
         config = ConfigParser()
+        config.optionxform = str
         config.read("./config/influx.ini")
         self._client = InfluxDBClient(url=config["INFLUX"]["url"], token=config["INFLUX"]["token"],
                                       org=config["INFLUX"]["org"])
@@ -83,35 +84,6 @@ class InfluxDataGetter:
         self._db = db
 
 # ----- PUBLIC METHODS
-  
-    def get_available_variables_at_time(self, time, chart_configs = None, user_config = None):
-        """
-        Gets the available variables (those that we can have a value for since the device has been installed on the instrument) at the given point in time.
-        Here, a variable means : SECOP module name + parameter. By default, this method returns the parameters "value" and "target", unless the config files used in chart_configs or user_config indicates other directives.
-
-        Parameters :
-            time (int) : the unix timestamps in seconds of the point in time to get the variables at.
-            chart_configs ([ChartConfig] | None) : an array of objects, each holding a configuration file for the chart. Configurations are applied in the order of the list.
-            user_config ({(str):{"cat":(str), "color":(str), "unit":(str)}} | None) : the Python dict representing the user configuration, applied at the end. The key is <secop_module.parameter>.
-
-        Returns :
-            [{"tag":(str), "unit":(str), "curves":[{"name":(str), "label":(str), "color":(str)}]}] : a list of dictionnaries, each one representing
-            a block of curves with their name, their label and their color to display, grouped by their category if given or unit (in tag).
-        """
-
-        all_setup_info = self._get_all_setup_info_as_dict(time)
-
-        available_variables = self._extract_variables(all_setup_info)
-        if not chart_configs == None:
-            for chart_config in chart_configs:
-                available_variables = self._filter_params_with_config(available_variables, chart_config)
-        if not user_config == None:
-            available_variables = self._filter_params_with_user_config(available_variables, user_config)
-        available_variables = self._remove_variables_params_not_displayed(available_variables)
-        available_variables = self._remove_variables_params_wihout_param_float_and_split(available_variables, time)               
-        res = self._group_variables_by_cat_unit(available_variables)
-
-        return res
 
     def get_curves_in_timerange(self, variables, time, interval = None):
         """
@@ -199,18 +171,17 @@ class InfluxDataGetter:
             parameter = "value" if len(var_param) == 1 else var_param[1]
 
             # we need to rename the "_time" column to simply "time" in case we want binning because of the comparison done later in the "binned points with same timestamp" process.
-            # chr(34) is the double quote char, because we cannot escape them in a f string
             query = f"""
                 from(bucket: "{self._bucket}")
                     |> range(start: {times[0]}, stop: {times[1] + 1})
                     |> filter(fn : (r) => r._measurement == "{variable_name_for_query}")
                     |> filter(fn : (r) => r._field == "{parameter+"_float"}")
-                    {"|> aggregateWindow(every: duration(v: "+ str(self._seconds_to_nanoseconds(interval))+"), fn: last, createEmpty:false, timeDst:"+chr(34)+"binning_time"+chr(34)+")" if interval != "None" else ""}
+                    {f'|> aggregateWindow(every: duration(v: {self._seconds_to_nanoseconds(interval)}), fn: last, createEmpty:false, timeDst:"binning_time")' if interval != 'None' else ''}
                     |> map(fn: (r) => ({{r with relative: (  float(v: uint(v: {"r.binning_time" if interval != "None" else "r._time"}) - uint(v:{self._seconds_to_nanoseconds(times[0])}))  / 1000000000.0   )}}))
                     |> map(fn: (r) => ({{r with timestamp: float(v: uint(v: {"r.binning_time" if interval != "None" else "r._time"})) / 1000000000.0}}))
-                    {"|> rename(columns: {_time:"+chr(34)+"time"+chr(34)+"})" if interval != "None" else ""}
+                    {'|> rename(columns: {_time:"time"})' if interval != 'None' else ''}
                     |> drop(columns:["_start", "_stop", "_field"])
-                    |> pivot(rowKey:["relative", "timestamp", "expired"{", "+chr(34)+"time"+chr(34) if interval != "None" else ""}], columnKey: ["_measurement"], valueColumn: "_value")
+                    |> pivot(rowKey:["relative", "timestamp", "expired"{', "time"' if interval != "None" else ''}], columnKey: ["_measurement"], valueColumn: "_value")
                 """
             data_frame = self._db.query_data_frame(query)
 
@@ -322,75 +293,6 @@ class InfluxDataGetter:
     
 # ----- PRIVATE METHODS
 
-    def _get_all_setup_info_as_dict(self, time):
-        """
-        Gets the value of the field setup_info in the measurements nicos/se_main, nicos/se_stick, nicos/se_addons as an array of Python dicts.
-        Takes the last setup_info dict (for each measurement) known at time.
-        
-        Parameters
-            time (int) : the unix timestamps in seconds of the point in time to get the variables at.
-
-        Returns :
-            [{(str):((str), {...})}]: an array of the parsed "setup_info dict" of each measurements. The key is the secop_module prefixed with "se_", and the value is a tuple with its first value
-            being the type of Secop device for this module, and the value is too big to give its signature. Some tuple examples can be found under graphs/setup_info_examples.
-
-        """
-        measurements = ["nicos/se_main", "nicos/se_stick", "nicos/se_addons"]
-        res = []
-        for measurement in measurements:
-            query = f"""
-            from(bucket: "{self._bucket}")
-            |> range(start: 0, stop: {time + 1})
-            |> filter(fn: (r) => r._measurement == "{measurement}")
-            |> filter(fn: (r) => r._field == "setup_info")
-            |> last()
-            |> yield(name: "res")
-            """
-            tables = self._db.query(query)
-            for table in tables:
-                for record in table.records:
-                    res.append(ast.literal_eval(record.get_value()))
-        return res
-        
-    def _extract_variables(self, all_setup_info_dict):
-        """
-        Extracts relevant information out of the setup_info dict for each available variable in measurements nicos/se_main, nicos/se_stick, nicos/se_addons
-
-        Parameters :
-            all_setup_info_dict ([{(str):((str), {...})}]) : an array of the parsed "setup_info dict" of each measurements. The key is the secop_module prefixed with "se_", and the value is a tuple with its first value
-            being the type of Secop device for this module, and the value is too big to give its signature. Some tuple examples can be found under graphs/setup_info_examples.
-
-        Returns :
-            [{"name":(str), "label":(str), "params":{(str):{"cat":(str), "color":(str), "unit":(str)}}}] : an array of dictionnaries, each containing the Influx name of the corresponding variable out of the setup_info dict,
-            the label to display in the Web GUI, and a dictionnary of parameters (including value), which consist of dictionnares with the category ("*" for value and target, "-" else), the color (empty for the moment)
-            and the unit ("1" if not available or empty), indexed by the name of the parameter.
-
-        """
-        available_varirables = []
-        added_names = []
-        for setup_info_dict in all_setup_info_dict:
-            for (_, content) in setup_info_dict.items():
-                if content[0] != "nicos.devices.secop.devices.SecopDevice":
-                    name = self._transform_secop_module_name_to_influx(content[1]["secop_module"])
-                    if name not in added_names:
-                        value_unit = "1" if (not "unit" in content[1].keys() or content[1]["unit"] == "")  else content[1]["unit"]
-                        variable = {
-                            "name":name,
-                            "label":content[1]["secop_module"],
-                            "params":{"value":{"cat":"*", "color":"", "unit":value_unit}} # main value, shown by default
-                        }
-
-                        for param_name, param_content in content[1]["params_cfg"].items():
-                            param_unit = "1" if (not "unit" in param_content.keys() or param_content["unit"] == "") else param_content["unit"]
-                            variable["params"][param_name] = {
-                                "cat":"*" if param_name == "target" else "-", # target is also shown by default, not the other parameters
-                                "color":"",
-                                "unit":param_unit
-                            }
-                        available_varirables.append(variable)
-                        added_names.append(name)
-        return available_varirables
-    
     def _transform_secop_module_name_to_influx(self, secop_module_name):
         """
         Transforms the name of the variable available in the setup_info dict into the Influx name.

influxgraph.py

@@ -1,19 +1,23 @@
-import time
+from time import time as current_time
 import logging
 import json
 import io
 import uuid
-from influxdb import InfluxDB, InfluxDataGetter
+from configparser import ConfigParser
+from math import ceil
+from sehistory.influx import InfluxDBWrapper
 from colors import assign_colors_to_curves
 from chart_config import ChartConfig
 from base import Instrument, get_abs_time
 from secop import SecopClient, SecopInstrument
 
 
+def split_tags(tags):
+    return {k: v.split(',') for k, v in tags.items()}
+
+
 class InfluxGraph:
-    """
-    Class implementing the logic of the different routes that are called by
-    the client to retrieve graph data with InfluxDB.
+    """Class implementing the logic of the different routes that are called by the client to retrieve graph data with InfluxDB.
     
     Global constants :
         HISTORICAL (int) : value that represents the "historical" visualization mode, meaning that the
@@ -29,96 +33,99 @@ class InfluxGraph:
         livemode (int) : the type of visualization the user is currently in. Can be HISTORICAL, ACTUAL or LIVE.
         end_query (int) : the unix timestamp in seconds of the most recent requested point in time of the last query
             or update.
-        lastvalues ({(str):((int), (float))}) : a dictionnary where the keys are the variable names, and the values
+        last_values ({(str):((int), (float))}) : a dictionnary where the keys are the variable names, and the values
             are tuples, where the first value is the unix timestamp of the most recent value known for this variable,
             and the second value its corresponding value
-        variables ({(str):(str)}) : a dictionnary of the current available variables requested by the client.
+        variables ({(str):(str)}) : a dictionary of the current available variables requested by the client.
         The key is the InfluxDB name of the curve, and the value is its label in the GUI.
     """
     HISTORICAL = 0
     ACTUAL = 1
     LIVE = 2
 
-    def __init__(self, influx_data_getter, instrument):
-        self.influx_data_getter = influx_data_getter
-        self.chart_configs = [ChartConfig("./config/generic.ini"), ChartConfig(f"./config/{instrument}.ini")]
+    def __init__(self, instrument):
+        self.instrument = instrument
+        self.db = instrument.db
+        instrument_name = instrument.title
+        # self.influx_data_getter = influx_data_getter
+        self.chart_configs = [ChartConfig("./config/generic.ini")]
+        try:
+            self.chart_configs.append(ChartConfig(f"./config/{instrument_name}.ini"))
+        except KeyError:
+            pass
         self.livemode = self.HISTORICAL
-        self.end_query = 0
-        self.lastvalues = {}
-        self.variables = {}  # name:label
-
-    def complete_to_end_and_feed_lastvalues(self, result, endtime):
-        """
-        Completes the data until the last requested point in time by adding the last known y-value at the end point.
-        Also feeds self.lastvalues.
-
-        Parameters :
-            result ({(str):[[(int),(float)]]}) : a dictionnary with the variable names as key, and an array of points,
-            which are a tuple (timestamp, y-value as float)
-            endtime (int) : the unix timestamp in seconds of the time we want to have data until
-        """
-        for var, c in result.items():
-            if c:
-                lastt, lastx = c[-1]
-                if lastt < endtime:
-                    c.append((endtime, lastx))
-                self.lastvalues[var] = (lastt, lastx)
+        self.last_values = {}  # dict <variable> of last known point (<time>, <value>)
+        self.last_time = {}  # dict <stream> of last received time
+        self.last_minute = 0
+        self.last_update = 0  # time of last call with a result
+        self.tags = {}  # tags for query (determines device and/or server)
 
     def w_graph(self, variables, time="-1800,0", interval=None):
-        """
-        Gets the curves given by variables in the time range "time", spaced by "interval" if given (binning/resolution)
+        """Get the curves given by variables in the time range "time"
+
+        spaced by "interval" if given (binning/resolution)
         Called when the route /graph is reached.
 
         Parameters :
-            variables (str) : a comma separataed value string of variable names (influx names) to retrieve
-            time (str) : a commma separated value string (range) of seconds. They are treated as relative from now
-                if they are lesser than one year.
-            interval (str) : the interval (resolution) of the values to get (string in milliseconds)
+            variables (str) : a comma separated string of variable names (influx names) to retrieve
+            time (str) : a commma separated value string (range) of seconds.
+                values < one year are treated as relative from now.
+            interval (str) : the interval (resolution) of the values to get (string in seconds)
 
         Returns :
-            {"type":"graph-draw", "graph":{(str):[[(int),(float)]]}} : a dictionnary with its "graph-draw" type
-                (so it can be processed by the client), and a "graph" dictionnary with the variable names as key,
+            {"type":"graph-draw", "graph":{(str):[[(int),(float)]]}} : a dictionary with its "graph-draw" type
+                (so it can be processed by the client), and a "graph" dictionary with the variable names as key,
                 and an array of points as a tuple (timestamp, y-value as float)
         """
-        time = [float(t) for t in time.split(',')]
-        start, end, now = get_abs_time(time + [0])
-        start, end, now = int(start), int(end), int(now)
-        queried_time_range = [start, end]
+        start, end, now = get_abs_time([float(t) for t in time.split(',')] + [0])
+        start, end, now = int(start), ceil(end), ceil(now)
         queried_variables = variables.split(',')
         self.livemode = self.ACTUAL if end+10 >= now else self.HISTORICAL
         logging.info('LIVE %g %g %d %d', end, now, end >= now, self.livemode)
-        if interval : interval = int(interval)
+        if interval:
+            interval = float(interval)
+        result = self.db.curves(start, end, queried_variables, merge='_measurement',
+                                interval=interval or None, **self.tags)
+        self.update_last(result)
+        self.db.complete(result, self.last_time, 'stream')
+        self.last_minute = now // 60
+        return dict(type='graph-draw', graph={k: v for k, v in result.items()})
 
-        result = self.influx_data_getter.get_curves_in_timerange(queried_variables, queried_time_range, interval)
-        self.complete_to_end_and_feed_lastvalues(result, min(end, now))
-        self.end_query = end
-
-        return dict(type='graph-draw', graph=result)
+    def update_last(self, curve_dict):
+        """update last values per variable and last time per stream"""
+        for key, curve in curve_dict.items():
+            stream = curve.tags.get('stream')
+            tlast, value = curve[-1]
+            self.last_values[key] = curve[-1]
+            self.last_time[stream] = max(self.last_time.get(stream, 0), tlast)
 
     def w_gettime(self, time):
-        """ 
-        Gets the server time for the give time.
+        """Get the server time for the given time(range).
+
         Called when the route /gettime is reached.
 
         Parameters :
-            time (str="-1800,0") : the given point in time represented by a string, which is a comma separated unix
-            timestamp values list (in seconds). They are treated as relative from now if they are lesser than one year.
+            time (str="-1800,0") : the given point in time represented by a string,
+            which is a comma separated unix timestamp values list (in seconds).
+            values < one year are treated as relative from now.
 
         Returns :
-            {"type":"time", "time":(int)} : a dictionnary with its "time" type (so the data can be processed by the
+            {"type":"time", "time":(int)} : a dictionary with its "time" type (so the data can be processed by the
                client) and the server unix timestamp in seconds corresponding to the time asked by the client
         """
-        time = [float(t) for t in time.split(',')]
-        return dict(type='time', time=get_abs_time(time))
+        return dict(type='time', time=get_abs_time(
+            [float(t) for t in time.split(',')]))
 
-    def w_getvars(self, time, userconfiguration = None):
-        """
-        Gets the curve names available at a given point in time, with a possible user configuration on the client side.
+    def w_getvars(self, time, userconfiguration=None, instrument=None, **tags):
+        """Get the curve names available at a given point in time
+
+        with a possible user configuration on the client side.
         Called when the route /getvars is reached.
         
         Parameters :
             time (str) : the given point in time represented by a string, which is a unix timestamp in seconds.
-                It is treated as relative from now if it is lesser than one year.
+                values < one year are treated as relative from now.
+                Might also be a comma separated time range.
             userconfiguration (str|None) : the JSON string representing the user configuration
 
         Returns :
@@ -130,25 +137,118 @@ class InfluxGraph:
             category or unit if absent) and their unit (in "blocks")
         """
         
-        time = [float(t) for t in time.split(',')]
-        end_time = int(get_abs_time(time)[-1])
-
-        if not userconfiguration == None : userconfiguration = json.loads(userconfiguration)
-        
-        blocks = self.influx_data_getter.get_available_variables_at_time(end_time, self.chart_configs, userconfiguration)
-        device_name = self.influx_data_getter.get_device_name(end_time)
-        # updates the self.variables attribute to keep track of the available variables
-        self.variables = {variable["name"]:variable["label"] for block in blocks for variable in block["curves"]}
+        time = get_abs_time([float(t) for t in time.split(',')])
+        start_time = int(time[0])
+        end_time = int(time[-1])
+        if userconfiguration is not None:
+            userconfiguration = json.loads(userconfiguration)
 
+        self.tags = split_tags(tags)
+        if instrument:
+            tags['stream'] = list(self.db.get_streams(instrument))
+        print('GETAV', self.tags)
+        blocks = self.get_available_variables(start_time, end_time, self.chart_configs, userconfiguration)
+        device_name = tags.get('device', '<unknown>')
+        # initialize self.last_values to keep track of the available variables
+        self.last_values = {var["name"]: [0, None] for block in blocks for var in block["curves"]}
         assign_colors_to_curves(blocks)
         result = dict(type='var_list')
         result['blocks'] = blocks
         result['device'] = device_name
+        print('DEVICE', device_name, tags)
+        for block in blocks:
+            print(block['tag'], [c['name'] for c in block['curves']])
         return result
 
-    def w_updategraph(self):
+    def get_available_variables(self, start_time, end_time, chart_configs=None, user_config=None):
+        """Gets the available variables
+
+        (those that we can have a value for since the device has been installed
+        on the instrument) at the given point in time.
+        Here, a variable means : SECOP module name + parameter.
+        By default, this method returns the parameters "value" and "target",
+        unless the config files used in chart_configs or user_config indicates other directives.
+
+        Parameters :
+            start_time, send_time (int) : the unix timestamps in seconds of the point in time to get the variables at.
+            chart_configs ([ChartConfig] | None) :
+               an array of objects, each holding a configuration file for the chart.
+               Configurations are applied in the order of the list.
+            user_config ({(str):{"cat":(str), "color":(str), "unit":(str)}} | None) :
+               the Python dict representing the user configuration, applied at the end.
+               The key is <secop_module.parameter>.
+
+        Returns :
+            [{"tag":(str), "unit":(str), "curves":[{"name":(str), "label":(str), "color":(str)}]}] :
+                a list of dictionnaries, each one representing
+            a block of curves with their name, their label and their color to display,
+            grouped by their category if given or unit (in tag).
         """
-        Sets the current visualisation mode to LIVE if not in HISTORICAL mode.
+        if start_time == end_time:
+            start_time = end_time - 3600
+        result = self.db.curves(start_time, end_time, _measurement=None, _field='float', **self.tags)
+        assert all(c.key_names[0] == '_measurement' for c in result.values())
+        variables = {k[0] for k in result}
+        config = {}
+        if chart_configs:
+            for chart_config in chart_configs:
+                for key, cfg in chart_config.variables.items():
+                    config.setdefault(key, {}).update(cfg)
+        if user_config:
+            for key, cfg in user_config.items():
+                config.setdefault(key, {}).update(cfg)
+
+        groups = {}
+
+        def add_to_groups(name, cat=None, unit='1', color='', label=None):
+            if cat == '-':
+                return
+            if name.endswith('.value'):
+                if not cat:
+                    cat = '*'
+                if not label:
+                    label = name[:-6]
+            elif name.endswith('.target'):
+                if not cat:
+                    cat = '*'
+            elif not cat:
+                return
+            tag = cat.replace('*', unit)
+            grp = groups.get(tag)
+            if grp is None:
+                curves = []
+                groups[tag] = {'tag': cat.replace('*', unit), 'unit': unit, 'curves': curves}
+            else:
+                curves = grp['curves']
+            curves.append({'name': name, 'unit': unit, 'label': label or name})
+
+        # treat variables in config first (in their order!)
+        result = {}
+        for key, cfg in config.items():
+            cat = cfg.pop('cat', None)
+            unit = cfg.get('unit', '1')
+            if '.' in key:
+                if key in variables:
+                    add_to_groups(key, cat, **cfg)
+                    variables.discard(key)
+            else:
+                var = f'{key}.value'
+                if var in variables:
+                    label = cfg.pop('label', None) or key
+                    add_to_groups(var, cat, label=label, **cfg)
+                    variables.discard(var)
+                var = f'{key}.target'
+                if var in variables:
+                    cfg.pop('color', None)
+                    add_to_groups(var, cat, **cfg)
+                    variables.discard(var)
+        for var in variables:
+            add_to_groups(var)
+        return list(groups.values())
+
+    def w_updategraph(self):
+        """Set the current visualisation mode to LIVE if not in HISTORICAL mode.
+
         Called when the route /updategraph is reached.
         Returns :
             {"type":"accept-graph", "live": bool} : a dict with its "accept-graph" type and a "live"
@@ -161,7 +261,7 @@ class InfluxGraph:
             self.livemode = self.LIVE
             return dict(type='accept-graph', live=True)
 
-    def w_export(self, variables, time, nan, interval):
+    def w_export(self, variables, time, nan, interval, timeoffset=None):
         """
         Returns the bytes of a dataframe with the curves given by variables in the time range "time"
         Called when the route /export is reached.
@@ -176,19 +276,15 @@ class InfluxGraph:
             io.BytesIO : an BytesIO object containing the dataframe to retrieve
         """
 
-        time = [float(t) for t in time.split(',')]
-        start, end = get_abs_time(time)
-        start, end = int(start), int(end)
+        start, end = get_abs_time([float(t) for t in time.split(',')])
+        start, end = int(start), ceil(end)
 
         queried_variables = variables.split(',')
-        if interval != "None" : interval = int(interval)
-       
-        df = self.influx_data_getter.get_curves_data_frame(queried_variables, [start, end], interval, self.variables)
-
-        mem = io.BytesIO()
-        df.to_csv(mem, sep="\t", index=False, float_format="%.15g", na_rep=nan)
-        mem.seek(0)
-        return mem
+        interval = float(interval) if interval else None
+        timeoffset = None if timeoffset == 'now' else (timeoffset or 0)
+        result = self.db.export(start, end, queried_variables, timeoffset=timeoffset, none=nan, interval=interval,
+                                **self.tags)
+        return io.BytesIO(result.encode('utf-8'))
 
     def graphpoll(self):
         """
@@ -199,45 +295,65 @@ class InfluxGraph:
 
         Returns :
             {"type":"graph-update", "time":(int), "graph":{(str):[[(int),(float)]]}} | None :
-                a dictionnary with its "graph-update" type
-                (so it can be processed by the client), and a "graph" dictionnary with the variable names as key,
+                a dictionary with its "graph-update" type
+                (so it can be processed by the client), and a "graph" dictionary with the variable names as key,
                 and an array of points, which are an array containing the timestamp
                 as their first value, and the y-value in float as their second one
         """
         if self.livemode != self.LIVE:
             return None
-        now, = get_abs_time([0])
-
-        result = self.influx_data_getter.poll_last_values(list(self.variables.keys()), self.lastvalues, now)
-        for variable, values in list(result.items()):
-            tlast = self.lastvalues.get(variable, (0,))[0]
-            # removes points older than the last known point
-            # (queries are in seconds and might return points already displayed)
-            while values and values[0][0] <= tlast:
-                values.pop(0)
-            if values and values[-1][0] > tlast:
-                self.lastvalues[variable] = values[-1]
+        now = current_time()
+        if now < int(self.last_update) + 1.5:
+            # the server is only waiting after a None return
+            # this avoids to many queries with expected empty result
+            return None
+        last_time = int(min(self.last_time.values()))
+        # if len(self.last_time) > 1:
+        #     print('time_poll_jitter', max(self.last_time.values()) - min(self.last_time.values()))
+        prev_minute, self.last_minute = self.last_minute, now // 60
+        fullminute = prev_minute != self.last_minute
+        add_prev = 3600 if fullminute else 0
+        result = self.db.curves(last_time, None, list(self.last_values),
+                                merge='_measurement', add_prev=add_prev, **self.tags)
+        to_remove = {}
+        for key, curve in result.items():
+            tlast = self.last_values.get(key, [0])[0]
+            # remove points older than the last known point. this might happen for different reasons:
+            # - queries are rounded to seconds
+            # - clocks of different streams might not be synched
+            l = len(curve)
+            for i, row in enumerate(curve):
+                if row[0] > tlast:
+                    del curve[:i]
+                    break
             else:
-                del result[variable]
-        if int(now / 60) != int(self.end_query / 60):
-            # Update unchanged values every plain minute
-            for var, (_, lastx) in self.lastvalues.items():
-                if var not in result:
-                    result[var] = [(now, lastx)]
-        self.end_query = now
+                if not fullminute:
+                    to_remove[key] = l
+        self.update_last(result)
+        if fullminute:
+            self.db.complete(result, self.last_time, 'stream')
+        for key, length in to_remove.items():
+            curve = result[key]
+            if len(curve) > l:
+                del curve[:l]
+            else:
+                if fullminute:
+                    print('R', key)
+                result.pop(key)
+        # print('poll', sum(len(c) for c in result.values()), self.last_time)
         if len(result) > 0:
-            return dict(type='graph-update', time=now, graph=result)
+            self.last_update = now
+            return dict(type='graph-update', time=last_time, graph=result)
         return None
 
-
 class InfluxInstrument(Instrument):
 
     def __init__(self, instr_name, inst_config=None):
         super().__init__()
         self.db = InfluxDB()
-        self.influx_data_getter = InfluxDataGetter(self.db, instr_name)
+        # self.influx_data_getter = InfluxDataGetter(self.db, instr_name)
         self.title = instr_name
-        self.device = self.influx_data_getter.get_device_name(int(time.time()))
+        self.device = self.influx_data_getter.get_device_name(int(current_time()))
 
     def new_client(self):
         return self.register(InfluxClient(self))
@@ -268,7 +384,7 @@ class InfluxParams:
 class InfluxClient(InfluxParams, InfluxGraph):
     def __init__(self, instrument):
         InfluxParams.__init__(self)
-        InfluxGraph.__init__(self, instrument.influx_data_getter, instrument.title)
+        InfluxGraph.__init__(self, instrument)
 
     def poll(self):
         messages = self.queue
@@ -282,7 +398,7 @@ class InfluxClient(InfluxParams, InfluxGraph):
 class SecopInfluxClient(SecopClient, InfluxGraph):
     def __init__(self, instrument):
         SecopClient.__init__(self, instrument)
-        InfluxGraph.__init__(self, instrument.influx_data_getter, instrument.title)
+        InfluxGraph.__init__(self, instrument)
 
     def poll(self):
         messages = super().poll()
@@ -296,10 +412,20 @@ class SecopInfluxInstrument(SecopInstrument):
 
     def __init__(self, inst_name, instrument_config):
         super().__init__(inst_name, instrument_config)
-        self.db = InfluxDB()
-        self.influx_data_getter = InfluxDataGetter(self.db, inst_name)
-        self.device = self.influx_data_getter.get_device_name(int(time.time()))
+        config = ConfigParser()
+        config.optionxform = str
+        config.read("./config/influx.ini")
+        section = config["INFLUX"]
+        self.db = InfluxDBWrapper('linse-c')
+        # self.db = InfluxDBWrapper(uri=section["url"], token=section["token"],
+        #                           org=section["org"], bucket=section['bucket'])
+        # self.influx_data_getter = InfluxDataGetter(self.db, inst_name)
+        # self.device = self.influx_data_getter.get_device_name(int(current_time()))
 
     def new_client(self):
         return self.register(SecopInfluxClient(self))
 
+    def get_stream_tags(self, timestamp=None):
+        return self.db.get_streams(None, timestamp)
+
+

secop.py

@@ -136,7 +136,7 @@ class SecopInstrument(Instrument):
         # test_day = instrument_config.get('test_day', None)
         # self.test_day = [int(x) for x in test_day.split('-')] if test_day else None
         self.title = inst_name
-        self.device = 'UNDEFINED'
+        self.device = ''
         self.nodes = []
         self.node_map = {}
         for host_port in host_ports.split(','):

webserver.py

@@ -165,8 +165,13 @@ def reply():
         logging.error('%s', traceback.format_exc())
         circularlog.log()
         msg = dict(type='error', request=path[1:], error=repr(e))
-    logging.info('REPLY %s %r', path, msg)
-    resp = flask.Response(json.dumps(msg), mimetype='application/json')
+    jsonmsg = json.dumps(msg)
+    if len(jsonmsg) < 120:
+        logging.info('REPLY %s %s', path, jsonmsg)
+    else:
+        logging.info('REPLY %s %s...', path, jsonmsg[:80])
+        logging.debug('REPLY %s %r', path, jsonmsg)
+    resp = flask.Response(jsonmsg, mimetype='application/json')
     resp.headers['Access-Control-Allow-Origin'] = '*'
     return resp
 
@@ -206,10 +211,34 @@ def replace_by_empty(file):
 
 
 @app.route('/')
-def default():
+def main():
     return general_file('SEAWebClient.html')
 
 
+@app.route('/select')
+def default():
+    out = ['''<html><body><table>
+<style>
+th {
+  text-align: left;
+}
+</style>
+<tr><th>Instrument</th><th colspan=99>Devices</th></tr>''']
+    result = {}
+    for stream, tags in instrument.get_stream_tags().items():
+        ins = tags.get('instrument', '0')
+        result.setdefault(ins, []).append((stream, tags.get('device')))
+    bare_streams = result.pop('0', [])
+    for ins, streams in result.items():
+        out.append(f'<tr><td><a href="/?ins={ins}">{ins}</a></td>')
+        out.extend(f'<td>{d or s}</td>' for s, d in streams)
+        out.append('</tr>')
+    for stream, device in bare_streams:
+        out.append(f'<tr><td><a href="/?srv={stream}">{stream}</a></td><td>{device}</td><tr>')
+    out.extend(['</table></body?</html>', ''])
+    return '\n'.join(out)
+
+
 @app.route('/<file>')
 def general_file(file):
     subdir = "client/"