doc: initial sphinx files.

Change-Id: I6943f2f39340cb6bad52996bbcf50710c0575b7b

.gitignore

@@ -8,3 +8,6 @@ pid/*
 
 RELEASE-VERSION
 build
+
+# Sphinx
+doc/srcdoc/_build

Makefile

@@ -12,6 +12,7 @@ clean:
 doc: doc/*.md
 	@echo "Generating html tree"
 	@bin/make_doc.py
+	$(MAKE) -C doc/srcdoc html
 
 demo:
 	@bin/secop-server -q demo &

doc/index.md

@@ -1,12 +0,0 @@
-Markdown docu to be generated
-=============================
-
-[Notes](notes.html)
-
-
-TODO's
-======
-see [here](todo.html)
-
-[Syntax hints](http://daringfireball.net/projects/markdown/syntax)
-

doc/open_questions.md

@@ -1,26 +0,0 @@
-
-  * Datatype for a mapping (like the calibration for the amagnet)
-  * How to set a Mapping or an enum (with all the name-value mappings) in the configfile?
-  * use toml vs. ini parser?
-
-  * error handling should be more consistent
-  * error-info stuff should be consistent
-
-  * dynamic device creation / modification (i.e. how to set the mapping of an enum during runtime?)
-
-  * propagation of units from main value to params
-
-  * switch params+cmds to 'accessibles'/attributes and keep in orderdDict?
-    -> needs Datatype for a function
-  * datatype for funcion: is argin a list anyway, or just one (structured) argument?
-    (so far we use a list and argout is a single datatype (which may be stuctured))
-
-  * polling: vendor specific or do we propose a common way to handle it?
-    (playground uses a per-device pollinterval. params may be polled less often)
-
-  * interface classes !!! (maybe with feature mixins like WindowTimeout?)
-
-  * hardware access: unify? how?
-
-  * demo-hardware?
-

doc/srcdoc/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = python -msphinx
+SPHINXPROJ    = SECoP
+SOURCEDIR     = source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/srcdoc/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=python -msphinx
+)
+set SOURCEDIR=source
+set BUILDDIR=_build
+set SPHINXPROJ=SECoP
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The Sphinx module was not found. Make sure you have Sphinx installed,
+	echo.then set the SPHINXBUILD environment variable to point to the full
+	echo.path of the 'sphinx-build' executable. Alternatively you may add the
+	echo.Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

doc/srcdoc/source/conf.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# SECoP documentation build configuration file, created by
+# sphinx-quickstart on Mon Sep 11 10:58:28 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import sys
+from os import path
+# sys.path.insert(0, os.path.abspath('.'))
+
+# Add import path for inplace usage
+sys.path.insert(0, path.abspath(path.join(path.dirname(__file__), '..', '..', '..')))
+
+from secop.version import get_version
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'SECoP'
+copyright = '2017, Dr. Enrico Faulhaber'
+author = 'Dr. Enrico Faulhaber'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The full version, including alpha/beta/rc tags.
+release = get_version()
+# The short X.Y version.
+version = release.split('-')[0]
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    '**': [
+        'about.html',
+        'navigation.html',
+        'relations.html',  # needs 'show_related': True theme option to display
+        'searchbox.html',
+        'donate.html',
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'SECoPdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'SECoP.tex', 'SECoP source documentation',
+     'Dr. Enrico Faulhaber', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'secop', 'SECoP source documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'SECoP', 'SECoP source documentation',
+     author, 'SECoP', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}

doc/srcdoc/source/index.rst

@@ -0,0 +1,17 @@
+Welcome to SECoP's documentation!
+=================================
+
+.. toctree::
+   :maxdepth: 2
+
+   protocol/index
+   playground/index
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

doc/srcdoc/source/playground/client/index.rst

@@ -0,0 +1,6 @@
+Client documentation
+====================
+
+.. toctree::
+   :maxdepth: 2
+

doc/srcdoc/source/playground/facility/demo/cryo.rst

@@ -0,0 +1,6 @@
+Demo cryostat
+=============
+
+.. automodule:: secop_demo.cryo
+    :members:
+

doc/srcdoc/source/playground/facility/demo/index.rst

@@ -0,0 +1,12 @@
+Demo
+====
+
+Specific sample environments
+----------------------------
+
+.. toctree::
+   :maxdepth: 3
+
+   cryo
+   test
+

doc/srcdoc/source/playground/facility/demo/test.rst

@@ -0,0 +1,6 @@
+Test devices
+=============
+
+.. automodule:: secop_demo.test
+    :members:
+

doc/srcdoc/source/playground/facility/ess/epics.rst

@@ -0,0 +1,6 @@
+EPICS modules
+=============
+
+.. automodule:: secop_ess.epics
+    :members:
+

doc/srcdoc/source/playground/facility/ess/index.rst

@@ -0,0 +1,11 @@
+ESS
+===
+
+Frameworks
+----------
+
+.. toctree::
+   :maxdepth: 3
+
+   epics
+

doc/srcdoc/source/playground/facility/index.rst

@@ -0,0 +1,9 @@
+Facility specific functionalities
+=================================
+
+.. toctree::
+   :maxdepth: 3
+
+   demo/index
+   mlz/index
+   ess/index

doc/srcdoc/source/playground/facility/mlz/amagnet.rst

@@ -0,0 +1,6 @@
+ANTARES magnet (amagnet)
+========================
+
+.. automodule:: secop_mlz.amagnet
+    :members:
+

doc/srcdoc/source/playground/facility/mlz/entangle.rst

@@ -0,0 +1,6 @@
+Entangle
+========
+
+.. automodule:: secop_mlz.entangle
+    :members:
+

doc/srcdoc/source/playground/facility/mlz/index.rst

@@ -0,0 +1,20 @@
+MLZ
+===
+
+Frameworks
+----------
+
+.. toctree::
+   :maxdepth: 3
+
+   entangle
+
+
+Specific sample environments
+----------------------------
+
+.. toctree::
+   :maxdepth: 3
+
+   amagnet
+

doc/srcdoc/source/playground/framework/datatypes.rst

@@ -0,0 +1,6 @@
+Datatypes
+=========
+
+.. automodule:: secop.datatypes
+    :members:
+

doc/srcdoc/source/playground/framework/errors.rst

@@ -0,0 +1,6 @@
+Exception classes
+=================
+
+.. automodule:: secop.errors
+    :members:
+

doc/srcdoc/source/playground/framework/index.rst

@@ -0,0 +1,9 @@
+Framework documentation
+=======================
+
+.. toctree::
+   :maxdepth: 2
+
+   datatypes
+   errors
+

doc/srcdoc/source/playground/gui/index.rst

@@ -0,0 +1,6 @@
+Graphical user interface documentation
+======================================
+
+.. toctree::
+   :maxdepth: 2
+

doc/srcdoc/source/playground/index.rst

@@ -0,0 +1,13 @@
+Playground documentation
+========================
+
+.. toctree::
+   :maxdepth: 2
+
+   server/index
+   client/index
+   framework/index
+   gui/index
+   facility/index
+
+

doc/srcdoc/source/playground/server/configuration.rst

@@ -0,0 +1,3 @@
+Configuration
+=============
+

doc/srcdoc/source/playground/server/index.rst

@@ -0,0 +1,11 @@
+Server documentation
+====================
+
+.. toctree::
+   :maxdepth: 3
+
+   starting
+   configuration
+   modules
+   protocol/index
+

doc/srcdoc/source/playground/server/modules.rst

@@ -0,0 +1,6 @@
+Module base classes
+===================
+
+.. automodule:: secop.modules
+    :members:
+

doc/srcdoc/source/playground/server/protocol/encoding/demo_v2.rst

@@ -0,0 +1,6 @@
+Demo V2
+=======
+
+.. automodule:: secop.protocol.encoding.demo_v2
+    :members:
+

doc/srcdoc/source/playground/server/protocol/encoding/demo_v3.rst

@@ -0,0 +1,6 @@
+Demo V3
+=======
+
+.. automodule:: secop.protocol.encoding.demo_v2
+    :members:
+

doc/srcdoc/source/playground/server/protocol/encoding/demo_v4.rst

@@ -0,0 +1,6 @@
+Demo V4
+=======
+
+.. automodule:: secop.protocol.encoding.demo_v2
+    :members:
+

doc/srcdoc/source/playground/server/protocol/encoding/demo_v5.rst

@@ -0,0 +1,6 @@
+Demo V5
+=======
+
+.. automodule:: secop.protocol.encoding.demo_v2
+    :members:
+

doc/srcdoc/source/playground/server/protocol/encoding/index.rst

@@ -0,0 +1,11 @@
+Encodings
+=========
+
+.. toctree::
+   :maxdepth: 3
+
+   demo_v2
+   demo_v3
+   demo_v4
+   demo_v5
+

doc/srcdoc/source/playground/server/protocol/framing/demo.rst

@@ -0,0 +1,7 @@
+Demo
+====
+
+.. automodule:: secop.protocol.framing.demo
+    :members:
+
+

doc/srcdoc/source/playground/server/protocol/framing/eol.rst

@@ -0,0 +1,6 @@
+EOL
+===
+
+.. automodule:: secop.protocol.framing.eol
+    :members:
+

doc/srcdoc/source/playground/server/protocol/framing/index.rst

@@ -0,0 +1,11 @@
+Framings
+========
+
+.. toctree::
+   :maxdepth: 3
+
+   demo
+   eol
+   null
+   rle
+

doc/srcdoc/source/playground/server/protocol/framing/null.rst

@@ -0,0 +1,6 @@
+Null
+====
+
+.. automodule:: secop.protocol.framing.null
+    :members:
+

doc/srcdoc/source/playground/server/protocol/framing/rle.rst

@@ -0,0 +1,6 @@
+RLE
+===
+
+.. automodule:: secop.protocol.framing.rle
+    :members:
+

doc/srcdoc/source/playground/server/protocol/index.rst

@@ -0,0 +1,10 @@
+Different protocols
+===================
+
+.. toctree::
+   :maxdepth: 3
+
+   encoding/index
+   framing/index
+   interface/index
+

doc/srcdoc/source/playground/server/protocol/interface/index.rst

@@ -0,0 +1,9 @@
+Interfaces
+==========
+
+.. toctree::
+   :maxdepth: 3
+
+   tcp
+   zmq
+

doc/srcdoc/source/playground/server/protocol/interface/tcp.rst

@@ -0,0 +1,6 @@
+TCP
+===
+
+.. automodule:: secop.protocol.interface.tcp
+    :members:
+

doc/srcdoc/source/playground/server/protocol/interface/zmq.rst

@@ -0,0 +1,6 @@
+ZMQ
+===
+
+.. automodule:: secop.protocol.interface.zmq
+    :members:
+

doc/srcdoc/source/playground/server/starting.rst

@@ -0,0 +1,21 @@
+Starting
+========
+
+The SECoP server can be started via the ``bin/secop-server`` script.
+
+.. parsed-literal::
+
+    usage: secop-server [-h] [-v | -q] [-d] name
+
+    Manage a SECoP server
+
+    positional arguments:
+      name             Name of the instance. Uses etc/name.cfg for configuration
+
+    optional arguments:
+      -h, --help       show this help message and exit
+      -v, --verbose    Output lots of diagnostic information
+      -q, --quiet      suppress non-error messages
+      -d, --daemonize  Run as daemon
+
+

doc/srcdoc/source/protocol/descjson.rst

@@ -0,0 +1,11 @@
+Structure of the descriptive json
+=================================
+
+* json = {"modules": <list_of_modules>, "properties": <list_of_sec-node_properties>, ...}
+* module = {"name": <name_of_module>, "parameters": <list_of_parameters>, "commands": <list_of_commands>, "properties": <list_of_module_properties>}
+* parameter = {"name": ..., "properties": <list_of_properties>}
+* command = {"name": ..., "properties": <list_of_properties>}
+* property = {"name":<name>, "datatype": <datatype>, "value": <value>}
+
+note: property may also be [<name>,<datatype>,<value>]
+

doc/srcdoc/source/protocol/descprop.rst

@@ -0,0 +1,48 @@
+Descriptive properties
+======================
+
+Mandatory descriptive properties
+--------------------------------
+
+parameter-properties
+++++++++++++++++++++
+
+* name (implicit)
+* datatype
+* readonly (bool)
+
+module-properties
++++++++++++++++++
+
+* interface_class [list_of_strings] (MAY be empty)
+
+SEC-Node-properties
++++++++++++++++++++
+
+* no mandatory properties
+
+Optional descriptive properties
+-------------------------------
+
+parameter-properties
+++++++++++++++++++++
+
+* unit (string), SHOULD be given if meaningful (if not given: unitless) (empty string: unit is one)
+* description (string), SHOULD be given
+* visibility
+* group (identifier) (MUST start with an uppercase letter) (if empty string: treat as not specified)
+
+module-properties
++++++++++++++++++
+
+* description (string), SHOULD be given
+* visibility
+* group (identifier) (MUST start with an uppercase letter) (if empty string: treat as not specified)
+* meaning ???
+* importance ???
+
+SEC-Node-properties
++++++++++++++++++++
+
+* description (string), SHOULD be given
+

doc/srcdoc/source/protocol/heartbeat.rst

@@ -0,0 +1,8 @@
+Heartbeat
+=========
+* ping gets an 'intended looptime' argument (as number in seconds or null to disable)
+* server replys as usual
+* if the server received no new message within twice the indended looptime, it may close the connection.
+* if the client receives no pong within 3s it may close the connection
+* later discussions showed, that the ping/pong should stay untouched and the keepalive time should be (de-)activated by a special message instead. Also the 'connection specific settings' from earlier drafts may be resurrected for this....
+

doc/srcdoc/source/protocol/hierarchy.rst

@@ -0,0 +1,9 @@
+Hirarchy
+========
+
+* group property (currently a identifier like string, may be extended to tree like substrucutres by allowing ':')
+* visibility (enum(3:expert, 2:advanced, 1:user)) (default to 1 if not given)
+  if visibility is set to user: everybody should see it
+  if visibility is set to advanced: advanced users should see it
+  if visibility is set to expert: only 'experts' should see it
+    

doc/srcdoc/source/protocol/history.rst

@@ -0,0 +1,66 @@
+History
+=======
+
+Meeting 29.5.2017
+-----------------
+
+* for api: float is 'double'
+* everything countable is int64_t
+* description is string (UTF-8 without embedded \0) (zero terminated for API)
+* names / identifiers are:  [_a-z][_a-z0-9]{0,63}
+* BLOB is [length, string_encoding (base64 or json_string) ] ???
+* enum is transferred by value (api: int64_t)
+* basic data types: string, BLOB(maxsize), int, double, bool, enum(mapping)
+* encode as ["string"] ["blob"] ["int"] ["double"] ["bool"] ["enum", {<number_value>:<name>}]
+* send as json_string [length, json_string] number number 0_or_1 number_value
+* complex data types: array, tuple, struct
+* encode as: ["array", <subtype>] ["tuple", [<list_of_compositing data types>] ["struct", {"name_of_subcomponent":<type of subcomponent>}]
+* send as [array] [array} {mapping}
+* forbid: structs in structs, nesting level > 3, arrays may only contain basic types + tuple
+* essential features should not rely on complex data types
+* fallback: if ECS can not handle a non-basic datatype: handle as string containing the JSON-representation.
+* mandatory for all ECS: enum, int, double, string, bool, tuple(enum,string)
+
+Merge datatype and validator
+++++++++++++++++++++++++++++
+
+* ["enum", {<number_value>:<json_string>}]
+* ["int"] or ["int", <lowest_allowed_value>, <highest_allowed_value>]
+* ["double"] or ["double", <lowest_allowed_value>, <highest_allowed_value>]
+* ["bool"]
+* ["blob", <maximum_size_in_bytes>] or ["blob", <minimum_size_in_bytes>, <maximum_size_in_bytes>]
+* ["string", <maximum_allowed_length>] or ["string", <min_size>, <max_size>]
+* ["array", <basic_data_type>, <max_elements>] or ["array", <dtype>, <min_elements>, <max_elements>]
+* ["tuple", [ <list_of_dtypes ]]
+* ["struct", { <name_of_component_as_json_string>:<dtype>}]
+
+Examples
+++++++++
+
+* status: ["tuple", [ ["enum", {0:"init", 100:"idle", 200:"warn", 300:"busy"}], ["string", 255] ] ]
+* p/pi/pid-triple: ["array", ["double", 0, 100], 1, 3]
+
+
+Meeting 30.5.2017
+-----------------
+
+* data values can be transferred as json_null, meaning: no value yet
+* json_null can not be used inside structured data types
+* property name for datatype is "datatype"
+
+Meeting 11.9.2017
+-----------------
+
+Merge datatype and validator
+++++++++++++++++++++++++++++
+
+  * enum, int, double, bool, tuple, struct as before
+  * ["blob", <maximum_size_in_bytes>] or ["blob", <maximum_size_in_bytes>, <minimum_size_in_bytes>]
+  * ["string", <maximum_allowed_length>] or ["string", <max_size_in_bytes>, <minimum_size_in_bytes>]
+  * ["array", <basic_data_type>, <max_elements>] or ["array", <dtype>, <max_elements>, <min_elements>]
+
+Interface_class
++++++++++++++++
+
+  * Drivable, Writable, Readable, Module (first character uppercase, no middle 'e')
+

doc/srcdoc/source/protocol/index.rst

@@ -0,0 +1,17 @@
+Protocol documentation
+======================
+
+.. toctree::
+   :maxdepth: 2
+
+   messages
+   descprop
+   hierarchy
+   descjson
+   timeformat
+   modsubset
+   heartbeat
+   todo
+   history
+
+

doc/srcdoc/source/protocol/messages.rst

@@ -1,4 +1,3 @@
-
 SECoP Messages
 ==============
 
@@ -33,7 +32,8 @@ Also: each client MUST at any time correctly handle incoming event messages, eve
 Implementation node:
 Both SEC-node and ECS-client can close the connection at any time!
 
-list of Intents/Actions:
+
+List of Intents/Actions:
 ------------------------
 
   * Identify -> Ident
@@ -55,6 +55,7 @@ Warning: One Message per line: Description line can be looooong!!!
 Allowed combinations as examples:
 (replace <..> with sensible stuff)
 
+
 Identify
 --------
 
@@ -64,6 +65,7 @@ Identify
   Format is intentionally choosen to be compatible to SCPI (for this query only).
   It is NOT intended to transport information about the manufacturer of the hardware, but to identify this as a SECoP device and transfer the protocol version!
 
+
 Describe
 --------
 
@@ -131,6 +133,7 @@ Heartbeat
 
 EVENT
 -----
+
 Events can be emitted any time from the SEC-node (except if they would interrupt another message).
 
   * Request: None. Events can be requested by Trigger or by Activating Async Mode.
@@ -179,10 +182,9 @@ ERROR
   * 'ERROR SyntaxError "meas:Volt?"
 
 
-Example
-=======
+Examples
+--------
 
-<pre>
 (client connects):
 (client)    '*IDN?'
 (SEC-node)  'Sine2020WP7.1&ISSE, SECoP, V2016-11-30, rc1'
@@ -197,166 +199,4 @@ Example
 (SEC-node)  'pong fancy_nonce_37'
 (SEC-node)  'update T1 [3.49,{"t":"149128945.921397","e":0.01897}]'
 ...
-</pre>
 
-Discussion & open Points
-========================
-
-  * If more than one connection exists: shall all events be relayed to all listeners?
-  * how about WRITE/COMMAND replies? Shall they go to all connected clients?
-  * structure of descriptive data needs to be specified
-  * same for JSON_stuff for Error Messages
-  * 'changed' may be 'readback'
-  * 'change' may be 'write'
-  * 'read' may be 'poll'
-  * the whole message may be json object (bigger, uglier to read)
-  * which events are broadcast or unicast?
-  * do we need a way to correlate a reply with a request?
-  * ...
-
-
-Meeting 29.5.2017
-=================
-  * for api: float is 'double'
-  * everything countable is int64_t
-  * description is string (UTF-8 without embedded \0) (zero terminated for API)
-  * names / identifiers are:  [_a-z][_a-z0-9]{0,63}
-  * BLOB is [length, string_encoding (base64 or json_string) ] ???
-  * enum is transferred by value (api: int64_t)
-
-  * basic data types: string, BLOB(maxsize), int, double, bool, enum(mapping)
-  * encode as ["string"] ["blob"] ["int"] ["double"] ["bool"] ["enum", {&lt;number_value&gt;:&lt;name&gt;}]
-  * send as json_string [length, json_string] number number 0_or_1 number_value
-  
-  * complex data types: array, tuple, struct
-  * encode as: ["array", &lt;subtype&gt;] ["tuple", [&lt;list_of_compositing data types&gt;] ["struct", {"name_of_subcomponent":&lt;type of subcomponent&gt;}]
-  * send as [array] [array} {mapping}
-  * forbid: structs in structs, nesting level &gt; 3, arrays may only contain basic types + tuple
-  * essential features should not rely on complex data types
-  * fallback: if ECS can not handle a non-basic datatype: handle as string containing the JSON-representation.
-
-  * mandatory for all ECS: enum, int, double, string, bool, tuple(enum,string)
-
-
-merge datatype and validator:
------------------------------
-  * ["enum", {&lt;number_value&gt;:&lt;json_string&gt;}]
-  * ["int"] or ["int", &lt;lowest_allowed_value&gt;, &lt;highest_allowed_value&gt;]
-  * ["double"] or ["double", &lt;lowest_allowed_value&gt;, &lt;highest_allowed_value&gt;]
-  * ["bool"]
-  * ["blob", &lt;maximum_size_in_bytes&gt;] or ["blob", &lt;minimum_size_in_bytes&gt;, &lt;maximum_size_in_bytes&gt;]
-  * ["string", &lt;maximum_allowed_length&gt;] or ["string", &lt;min_size&gt;, &lt;max_size&gt;]
-  * ["array", &lt;basic_data_type&gt;, &lt;max_elements&gt;] or ["array", &lt;dtype&gt;, &lt;min_elements&gt;, &lt;max_elements&gt;]
-  * ["tuple", [ &lt;list_of_dtypes ]]
-  * ["struct", { &lt;name_of_component_as_json_string&gt;:&lt;dtype&gt;}]
-
-examples:
-
-  * status: ["tuple", [ ["enum", {0:"init", 100:"idle", 200:"warn", 300:"busy"}], ["string", 255] ] ]
-  * p/pi/pid-triple: ["array", ["double", 0, 100], 1, 3]
-
-
-30.5.2017
-=========
-
-  * data values can be transferred as json_null, meaning: no value yet
-  * json_null can not be used inside structured data types
-  * property name for datatype is "datatype"
-
-
-Mandatory descriptive properties
-================================
-
-parameter-properties
---------------------
-  * name (implicit)
-  * datatype
-  * readonly (bool)
-
-module-properties
------------------
-  * interface_class [list_of_strings] (MAY be empty)
-
-SEC-Node-properties
--------------------
-  * no mandatory properties
-
-Optional descriptive properties
-===============================
-
-parameter-properties
---------------------
-  * unit (string), SHOULD be given if meaningful (if not given: unitless) (empty string: unit is one)
-  * description (string), SHOULD be given
-  * visibility
-  * group (identifier) (MUST start with an uppercase letter) (if empty string: treat as not specified)
-
-module-properties
------------------
-  * description (string), SHOULD be given
-  * visibility
-  * group (identifier) (MUST start with an uppercase letter) (if empty string: treat as not specified)
-  * meaning ???
-  * importance ???
-
-SEC-Node-properties
--------------------
-  * description (string), SHOULD be given
-
-
-Hirarchy
-========
-  * group property (currently a identifier like string, may be extended to tree like substrucutres by allowing ':')
-  * visibility (enum(3:expert, 2:advanced, 1:user)) (default to 1 if not given)
-    if visibility is set to user: everybody should see it
-    if visibility is set to advanced: advanced users should see it
-    if visibility is set to expert: only 'experts' should see it
-    
-structure of the descriptive json
-=================================
-
-  * json = {"modules": &lt;list_of_modules&gt;, "properties": &lt;list_of_sec-node_properties&gt;, ...}
-  * module = {"name": &lt;name_of_module&gt;, "parameters": &lt;list_of_parameters&gt;, "commands": &lt;list_of_commands&gt;, "properties": &lt;list_of_module_properties&gt;}
-  * parameter = {"name": ..., "properties": &lt;list_of_properties&gt;}
-  * command = {"name": ..., "properties": &lt;list_of_properties&gt;}
-  * property = {"name":&lt;name&gt;, "datatype": &lt;datatype&gt;, "value": &lt;value&gt;}
-
-note: property may also be [&lt;name&gt;,&lt;datatype&gt;,&lt;value&gt;]
-
-Timeformat
-==========
-  * format goes to 'timestamp since epoch as double with a resolution of at least 1ms'
-  * SEC-node-property: timestamp is accurate or relative
-  * Or: extend pong response contains the localtime (formatted as a timestamp)
-
-activate subset of modules
-==========================
-  * activate/deactivate may get an optional 2nd argument and work only on this.
-
-  * add equipment_id [_0-9a-zA-A]{4,} as SEC-node property (mandatory) (prefixed with ficility/manufacturer)
-  * change response to 'describe' to 'describing ALL <json_description>'
-
-  * '(de-)activate samething' -> '(de-)activated something'
-
-heartbeat
-=========
-  * ping gets an 'intended looptime' argument (as number in seconds or null to disable)
-  * server replys as usual
-  * if the server received no new message within twice the indended looptime, it may close the connection.
-  * if the client receives no pong within 3s it may close the connection
-  * later discussions showed, that the ping/pong should stay untouched and the keepalive time should be (de-)activated by a special message instead. Also the 'connection specific settings' from earlier drafts may be resurrected for this....
-
-
-11.9.2017
-=========
-
-merge datatype and validator:
------------------------------
-  * enum, int, double, bool, tuple, struct as before
-  * ["blob", &lt;maximum_size_in_bytes&gt;] or ["blob", &lt;maximum_size_in_bytes&gt;, &lt;minimum_size_in_bytes&gt;]
-  * ["string", &lt;maximum_allowed_length&gt;] or ["string", &lt;max_size_in_bytes&gt;, &lt;minimum_size_in_bytes&gt;]
-  * ["array", &lt;basic_data_type&gt;, &lt;max_elements&gt;] or ["array", &lt;dtype&gt;, &lt;max_elements&gt;, &lt;min_elements&gt;]
-
-interface_class
----------------
-  * Drivable, Writable, Readable, Module (first character uppercase, no middle 'e')

doc/srcdoc/source/protocol/modsubset.rst

@@ -0,0 +1,8 @@
+Activate subset of modules
+==========================
+
+* activate/deactivate may get an optional 2nd argument and work only on this.
+* add equipment_id [_0-9a-zA-A]{4,} as SEC-node property (mandatory) (prefixed with ficility/manufacturer)
+* change response to 'describe' to 'describing ALL <json_description>'
+* '(de-)activate samething' -> '(de-)activated something'
+

doc/srcdoc/source/protocol/timeformat.rst

@@ -0,0 +1,7 @@
+Timeformat
+==========
+
+* format goes to 'timestamp since epoch as double with a resolution of at least 1ms'
+* SEC-node-property: timestamp is accurate or relative
+* Or: extend pong response contains the localtime (formatted as a timestamp)
+

doc/srcdoc/source/protocol/todo.rst

@@ -0,0 +1,101 @@
+=========
+Todo List
+=========
+
+Open questions
+==============
+
+* Datatype for a mapping (like the calibration for the amagnet)
+* How to set a Mapping or an enum (with all the name-value mappings) in the configfile?
+* use toml vs. ini parser?
+* error handling should be more consistent
+* error-info stuff should be consistent
+* dynamic device creation / modification (i.e. how to set the mapping of an enum during runtime?)
+* propagation of units from main value to params
+* switch params+cmds to 'accessibles'/attributes and keep in orderdDict?
+  -> needs Datatype for a function
+* datatype for funcion: is argin a list anyway, or just one (structured) argument?
+  (so far we use a list and argout is a single datatype (which may be stuctured))
+* polling: vendor specific or do we propose a common way to handle it? 
+  (playground uses a per-device pollinterval. params may be polled less often)
+* interface classes !!! (maybe with feature mixins like WindowTimeout?)
+* hardware access: unify? how?
+* demo-hardware?
+* If more than one connection exists: shall all events be relayed to all listeners?
+* how about WRITE/COMMAND replies? Shall they go to all connected clients?
+* structure of descriptive data needs to be specified
+* same for JSON_stuff for Error Messages
+* 'changed' may be 'readback'
+* 'change' may be 'write'
+* 'read' may be 'poll'
+* the whole message may be json object (bigger, uglier to read)
+* which events are broadcast or unicast?
+* do we need a way to correlate a reply with a request?
+
+
+Todos
+=====
+
+Structure
+---------
+
+ * stronger structure insides src
+
+   * src/server for everything server related
+   * src/client for everything client related (ProxyDevice!)
+   * src/protocol for protocol specific things
+
+     * need subtree for different implementations to play with
+
+   * src/lib for helpers and other stuff
+
+ * possibly a parallel src tree for cpp version
+
+
+Client
+------
+
+ * maybe start with a python shell and some import magic
+ * later a GUI may be a good idea
+ * client: one connection for each device?
+ * another connection for async data?
+
+
+Server
+------
+
+ * rewrite MessageHandler to be agnostic of server
+ * move encoding to interface
+ * allow multiple interfaces per server
+ * fix error handling an make it consistent
+
+Device framework
+----------------
+
+ * supply properties for PARAMS to auto-generate async data units
+ * self-polling support
+ * generic devicethreads
+ * proxydevice
+ * make get_device uri-aware
+
+
+Testsuite
+---------
+
+ * need a testsuite (pytest)
+ * embedded tests inside the actual files grow difficult to maintain
+
+
+Documentation
+-------------
+
+ * transfer build docu into wiki via automated jobfile
+
+
+ 
+
+Transfer of blobs via json
+--------------------------
+
+ * use base64
+

doc/todo.md

@@ -1,55 +0,0 @@
-# TODO List #
-
-## Structure ##
-
- * stronger structure insides src
-   * src/server for everything server related
-   * src/client for everything client related (ProxyDevice!)
-   * src/protocol for protocol specific things
-     * need subtree for different implementations to play with
-   * src/lib for helpers and other stuff
- * possibly a parallel src tree for cpp version
-
-
-## A Client ##
-
- * maybe start with a python shell and some import magic
- * later a GUI may be a good idea
- * client: one connection for each device?
- * another connection for async data?
-
-
-## A Server ##
-
- * rewrite MessageHandler to be agnostic of server
- * move encoding to interface
- * allow multiple interfaces per server
- * fix error handling an make it consistent
-
-## Device framework ##
-
- * supply properties for PARAMS to auto-generate async data units
- * self-polling support
- * generic devicethreads
- * proxydevice
- * make get_device uri-aware
-
-
-## Testsuite ##
-
- * embedded tests inside the actual files grow difficult to maintain
-=> need a testsuite (pytest)
-
-
-## docu ##
-
- * mabe use sphinx to generate docu: a pdf can then be auto-generated....
- * transfer build docu into wiki via automated jobfile
-
-
- 
-
-## transfer of blobs via json ##
-
- * use base64
-

requirements-dev.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme