resolve problem with DefuncDevice

renew envlist even when not changed
if not, devices in envlist are not exchanged by new instances.

commands.py

@@ -22,7 +22,8 @@
 from nicos import session, config
 from nicos.utils import printTable
 from nicos.commands import helparglist, usercommand
-from servicemanager import FrappyManager
+from .devices import get_frappy_config, all_info
+from servicemanager import FrappyManager, SeaManager
 
 
 SERVICES = FrappyManager.services
@@ -30,57 +31,24 @@ SERVICES = FrappyManager.services
 
 @usercommand
 def set_se_list():
-    frappy_config = session.devices['frappy_config']
-    frappy_config.set_envlist()
+    fc = get_frappy_config()
+    if fc:
+        fc.set_envalias()
 
 
 @usercommand
-@helparglist('main [, stick [, addons]]')
-def frappy(*args, main=None, stick=None, addons=None, force=False):
-    """(re)start frappy server(s) with given configs and load setup if needed
+def frappy_main(*args):
+    raise NameError('frappy_main(<cfg>) is no longer avaiable, use frappy(<cfg>) instead')
 
-    - without argument: list running frappy servers, restart failed frappy servers
-    - frappy('<cfg>'): if available, the standard stick is added too
-    - frappy(''): the stick is removed too
-    - addons are not changed when not given
-    - frappy(main='<cfg>')      # main cfg is changed, but stick is kept
-    - frappy('restart')         # restart all frappy servers
-    - frappy(stick='restart')   # restart stick frappy server
-    """
-    confirmed = FrappyManager().cfg_from_sea(config.instrument).get('confirmed')
-    if args:
-        if main is not None:
-            raise TypeError('got multiple values for main')
-        main = args[0]
-        if len(args) == 1:  # special case: main given as single argument
-            if main == 'restart':
-                stick = 'restart'
-                addons = 'restart'
-            elif stick is None:  # auto stick
-                if main == '':
-                    stick = ''  # remove stick with main
-                else:
-                    allsticks = FrappyManager().all_cfg(config.instrument, 'stick')
-                    stickcfg = main + 'stick'
-                    if stickcfg in allsticks:
-                        # if a default stick is available, start this also
-                        stick = stickcfg
-                    else:
-                        stick = ''  # remove stick when main has changed
-        else:
-            if stick is not None:
-                raise TypeError('got multiple values for stick')
-            stick, *alist = args[1:]
-            if alist:
-                if addons is not None:
-                    raise TypeError('got multiple values for addons')
-                addons = ','.join(alist)
-    if confirmed and confirmed != main and main not in (None, 'restart') and not force:
-        session.log.warning('%r is plugged to the cryostat control rack', confirmed)
-        session.log.warning('if you are sure, use frappy(..., force=True)', confirmed)
-        raise TypeError('refuse to override plugged device')
-    frappy_config = session.devices['frappy_config']
-    frappy_config.show_config(*frappy_config.check_or_start(main, stick, addons))
+
+@usercommand
+def frappy_stick(*args):
+    raise NameError('frappy_stick(<cfg>) is no longer avaiable, use frappy(stick=<cfg>) instead')
+
+
+@usercommand
+def frappy_addons(*args):
+    raise NameError('frappy_addons(<cfg>) is no longer avaiable, use frappy(addons=<cfg>) instead')
 
 
 @usercommand
@@ -104,8 +72,3 @@ def frappy_list(service=None):
 
     FrappyManager().do_listcfg(config.instrument, service or 'main', prt)
     session.log.info('\n%s', '\n'.join(content))
-
-
-@usercommand
-def frappy_changed():
-    session.devices['frappy_config'].changed()

devices.py

@@ -1,4 +1,3 @@
-#  -*- coding: utf-8 -*-
 # *****************************************************************************
 # NICOS, the Networked Instrument Control System of the MLZ
 # Copyright (c) 2009-2018 by the NICOS contributors (see AUTHORS)
@@ -27,19 +26,22 @@ SEC Node with added functionality for starting and stopping frappy servers
 connected to a SEC node
 """
 
-import time
 import threading
+import socket
+import json
 from nicos import config, session
 from nicos.core import Override, Param, Moveable, status, POLLER, SIMULATION, DeviceAlias, \
-    Device, anytype, listof
+    Device, anytype, listof, MASTER
 from nicos.devices.secop.devices import SecNodeDevice
-from nicos.core.utils import USER, User, createThread
-from nicos.services.daemon.script import RequestError, ScriptRequest
+from nicos.core.utils import createThread
 from nicos.utils.comparestrings import compare
 from nicos.devices.secop.devices import get_attaching_devices
 from nicos.commands.basic import AddSetup, CreateAllDevices, CreateDevice
-from servicemanager import FrappyManager
+from nicos.utils import loggers
+from servicemanager import FrappyManager, SeaManager, Reconnect, Keep
 
+
+SECOP_UDP_PORT = 10767
 SERVICES = FrappyManager.services
 
 
@@ -58,7 +60,7 @@ def applyAliasConfig():
     """
     # reimplemented from Session.applyAliasConfig
     # apply also when target dev name does not change, as the target device might have
-    # be exchanged in the mean time
+    # be exchanged in the meantime
     for aliasname, targets in session.alias_config.items():
         if aliasname not in session.devices:
             continue  # silently ignore
@@ -86,32 +88,48 @@ def cleanup_defunct():
 
 
 def all_info(all_cfg, prefix='currently configured: '):
-    info = []
     addkwd = False
+    info = []
     for srv in SERVICES:
-        cfginfo = all_cfg.get(srv)
-        if cfginfo is None:
+        cfglist = all_cfg.get(srv)
+        if cfglist is None:
             addkwd = True
-        elif addkwd:
-            info.append('%s=%r' % (srv, cfginfo))
         else:
-            info.append(repr(cfginfo))
+            # if cfglist is True:
+            #     cfglist = ['reconnect']
+            if isinstance(cfglist, str):
+                cfglist = [cfglist]
+            cfginfo = ','.join(c if isinstance(c, str)
+                               else f"<one of {', '.join(repr(v) for v in c)}>"
+                               for c in cfglist)
+            if addkwd:
+                info.append('%s=%r' % (srv, cfginfo))
+            else:
+                info.append(repr(cfginfo))
     return f"{prefix}frappy({', '.join(info)})"
 
 
-def get_guess(allcfg):
-    guess = {}
-    seacfg = FrappyManager().cfg_from_sea(config.instrument)
-    guess.pop('confirmed', '')
-    for s in ('main', 'stick', 'addons'):
-        info = allcfg.get(s, '')
-        prev = info.split(' ', 1)[0]
-        if prev != info:
-            guess[s] = prev
-        fromsea = seacfg.get(s)
-        if fromsea and fromsea != prev:
-            guess[s] = fromsea
-    return guess
+def get_frappy_config():
+    try:
+        return session.devices['frappy']
+    except KeyError:
+        session.log.exception("the frappy device is not available - 'frappy' setup is not loaded")
+        return None
+
+
+def send_other_udp(uri, instrument, device=None):
+    """inform the feeder about the start of a frappy server"""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+    msg = {
+        'SECoP': 'for_other_node',
+        'uri': uri,
+        'instrument': instrument,
+    }
+    if device:
+        msg['device'] = device
+    msg = json.dumps(msg, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
+    sock.sendto(msg, ('255.255.255.255', SECOP_UDP_PORT))
 
 
 class FrappyConfig(Device):
@@ -137,36 +155,137 @@ class FrappyConfig(Device):
             type=listof(str), default=[]),
     }
 
-    meanings = list(parameters)
-    meanings.remove('nodes')
-    _trigger_change = None
-    _previous_shown = None
+    meanings = [n for n, p in parameters.items() if p.type is anytype]
+    _update_setup = 'on_frappy'
+    _back_to_normal = None
+    _initial_info = None
+    _shutdown_event = None
+    _restarting = False
+    _within_update_setup = False
+    _current_cfgs = None
+    _target_cfgs = None
+    _rebuild_env = None
 
     def doInit(self, mode):
         if mode != SIMULATION and session.sessiontype != POLLER:
-            self._trigger_change = threading.Event()
-            createThread('frappy change notification', self.handle_notifications)
+            self._shutdown_event = threading.Event()
+            for name in self.nodes:
+                secnode = session.devices.get(name)
+                if secnode:
+                    secnode.uri = ''
+            createThread('check frappy and sea servers', self.detect_changes)
 
-    def handle_notifications(self):
-        controller = session.daemon_device._controller
-        while True:
-            self._trigger_change.wait()
-            self._trigger_change.clear()
-            time.sleep(2)
-            if self._trigger_change.is_set():
+    def doShutdown(self):
+        self._shutdown_event.set()
+
+    def detect_changes(self):
+        before_check = before_change = prev_shown = None
+        cnt = 0
+        self._back_to_normal = None
+        while not self._shutdown_event.wait(1):
+            busy = session.daemon_device._controller.status >= 0
+            if self._restarting or (busy and cnt < 10):
+                # do not check while restarting and check only every 10 sec when busy
+                cnt += 1
                 continue
-            try:
-                current = self.check_or_start()
-                if current != self._previous_shown:
-                    cmd = 'frappy_changed()'
-                    controller.new_request(ScriptRequest(cmd, None, User('guest', USER)))
-            except RequestError as e:
-                session.log.error(f'can not queue request {e!r}')
+            if not busy and self._back_to_normal:
+                session.log.info('  %s', 75*'_')
+                session.log.info(' ')
+                session.log.info('  sample environment servers match configuration:')
+                session.log.info(all_info(self._current_cfgs, '  '))
+                session.log.info('  %s', 75*'_')
+                self._back_to_normal = None
+            cnt = 0
+            need_change, changes, fm = to_consider = self.to_consider()
+            if fm.state == before_change:
+                continue
+            if not need_change:
+                if self._back_to_normal == 0:
+                    # we had a change, but are back to normal state
+                    self._back_to_normal = True
+                continue
+            if fm.state != before_check:
+                # must be twice the same
+                before_check = fm.state
+                continue
+            if busy or fm.state == prev_shown:
+                continue
+            self._back_to_normal = 0  # indicates that we had a change
+            self.show(True, to_consider)
+            prev_shown = fm.state
 
-    def check_or_start(self, main=None, stick=None, addons=None):
+    def to_consider(self, cfgs=None):
+        """return info about a proposed changes
+
+        :param cfgs: previous configuration
+        :return: <'need change' flag>, <change dict>, <frappy manager instance>
+
+        the information takes into account running frappy and sea servers
+        and their configuration
+
+        side effect: try to reconnect a frappy server which should run with
+        the target cfg
+        """
+        current_cfgs, target_cfgs = self.check_services()
+        if cfgs is None:
+            cfgs = current_cfgs
+        fm = FrappyManager()
+        proposed = fm.get_server_state(config.instrument, cfgs)
+        # if fm.error:
+        #     self.log.error('%s', fm.error)
+        changes = dict(proposed)
+        need_change = False
+        for service in SERVICES:
+            cfg = changes.get(service)  # proposed cfg
+            running = fm.frappy_cfgs.get(service)  # running cfg
+            if running:
+                prop = changes.get(service)
+                if prop:
+                    if prop == target_cfgs.get(service) == running:
+                        secnode = session.devices.get('se_' + service)
+                        if secnode:
+                            if secnode._secnode:
+                                secnode._secnode.connect()
+                            else:
+                                secnode._connect()
+                else:
+                    changes[service] = ''
+                    need_change = True
+                    cfg = ''
+            else:
+                if cfg == '':
+                    changes.pop(service)
+                    cfg = None
+                    if target_cfgs.get(service, '-') not in ('', '-'):
+                        need_change = 1
+            if cfg and not isinstance(cfg, Keep):
+                need_change = cfg
+        if not need_change and all(isinstance(v, Keep) for v in changes.values()):
+            self._rebuild_env = self.check_envalias()
+            if self._rebuild_env:
+                need_change = True
+        else:
+            self._rebuild_env = None
+        return need_change, changes, fm
+
+    def check_services(self):
+        cfgs = {}
+        targets = {}
+        for secnodename in self.nodes:
+            secnode = session.devices.get(secnodename)
+            if secnode:
+                cfgs[secnode.service] = secnode.get_info()
+                targets[secnode.service] = secnode.target
+        self._current_cfgs = cfgs
+        self._target_cfgs = targets
+        return cfgs, targets
+
+    def start_services(self, main=None, stick=None, addons=None,):
         """start/stop  frappy servers
 
-        for example: check_or_start(main='xy', stick='')
+        :param main, stick, addons: cfg for frappy servers, '' to stop, None to keep
+
+        for example: start_services(main='xy', stick='')
         - restart main server with cfg='xy'
         - stop stick server
         - do not touch addons server
@@ -174,50 +293,49 @@ class FrappyConfig(Device):
         in addition, if a newly given cfg is already used on a running server,
         this cfg is removed from the server (remark: cfg might be a comma separated list)
         """
-        services = {'main': main, 'stick': stick, 'addons': addons}
-        for service, cfg in services.items():
-            if cfg == '':
-                seaconn = session.devices.get(f'se_sea_{service}')
-                if seaconn and seaconn._attached_secnode:
-                    seaconn.communicate('frappy_remove %s' % service)
-        used_cfg = {}
-        all_cfg = {}
-        new_cfg = {}
-        secnodes = {}
-        remove_cfg = []
-        for service, cfginfo in services.items():
-            secnodes[service] = secnode = session.devices.get('se_' + service)
-            chkinfo = ''
-            if secnode:
-                all_cfg[service] = chkinfo = secnode.get_info()
-            if cfginfo is not None:
-                new_cfg[service] = chkinfo = cfginfo
+        self._restarting = True
+        try:
+            services = {'main': main, 'stick': stick, 'addons': addons}
+            for service, cfg in services.items():
+                if cfg == '':
+                    seaconn = session.devices.get(f'se_sea_{service}')
+                    if seaconn and seaconn._attached_secnode:
+                        try:
+                            seaconn.communicate('frappy_remove %s' % service)
+                        except Exception:
+                            pass
+            used_cfg = {}
+            all_cfg = {}
+            new_cfg = {}
+            secnodes = {}
+            remove_cfg = []
+            for service, cfginfo in services.items():
+                secnodes[service] = secnode = session.devices.get('se_' + service)
+                chkinfo = ''
+                if secnode:
+                    all_cfg[service] = chkinfo = secnode.get_info()
+                if cfginfo is not None and (cfginfo != chkinfo or not isinstance(cfginfo, Reconnect)):
+                    new_cfg[service] = chkinfo = cfginfo
 
-            # check cfg is not used twice
-            for cfg in chkinfo.split(','):
-                cfg = cfg.strip()
-                if cfg and cfg != 'restart':
-                    prev = used_cfg.get(cfg)
-                    if prev:
-                        raise ValueError('%r can not be used in both %s and %s' % (cfg, prev, service))
-                    used_cfg[cfg] = service
+                # check cfg is not used twice
+                for cfg in chkinfo.split(','):
+                    cfg = cfg.strip()
+                    if cfg and cfg != 'restart':
+                        prev = used_cfg.get(cfg)
+                        if prev:
+                            raise ValueError('%r can not be used in both %s and %s' % (cfg, prev, service))
+                        used_cfg[cfg] = service
 
-        for service, cfginfo in reversed(list(new_cfg.items())):
-            if cfginfo != all_cfg.get(service, ''):
+            for service, cfginfo in reversed(list(new_cfg.items())):
                 secnode = secnodes[service]
                 if secnode:
                     secnode('')  # stop previous frappy server
 
-        if new_cfg:
-            for service, cfginfo in new_cfg.items():
-                nodename = 'se_' + service
-                secnode = secnodes[service]
-                prev = all_cfg.get(service)
-                if cfginfo != prev:
-                    if cfginfo == 'restart':
-                        cfginfo = prev
-                        if not cfginfo:
-                            continue
+            if new_cfg:
+                for service, cfginfo in new_cfg.items():
+                    nodename = 'se_' + service
+                    secnode = secnodes[service]
+                    prev = all_cfg.get(service)
                     if not secnode:
                         if not cfginfo:
                             continue
@@ -226,31 +344,162 @@ class FrappyConfig(Device):
                     secnode(cfginfo)
                     all_cfg[service] = secnode.get_info()
                     CreateDevice(nodename)
-            cleanup_defunct()
-            CreateAllDevices()
-        self.set_envlist()
-        for secnode in remove_cfg:
-            secnode.disable()
-        return all_cfg, get_guess(all_cfg)
+                cleanup_defunct()
+                CreateAllDevices()
+            fm = FrappyManager()
+            ins = config.instrument
+            fm.get_server_state(ins, new_cfg)
+            recorders = {}
+            for service, secnode in secnodes.items():
+                if services.get(service) and secnode:
+                    cfg = fm.frappy_cfgs.get(service)
+                    seacfg = fm.frappy2sea.get(cfg)
+                    if secnode() and not seacfg:
+                        if cfg:
+                            recorders[service] = f'localhost:{fm.info[ins].get(service, 0)}/{cfg}'
+                        else:
+                            recorders[service] = secnode.uri
+                    secnode._secnode.connect()
+            if recorders:
+                try:
+                    fm.sea.sea_recorder(ins, recorders)
+                except Exception:
+                    pass
+            self.set_envalias()
+            for secnode in remove_cfg:
+                secnode.disable()
+        finally:
+            self._restarting = False
+        return all_cfg
 
-    def show_config(self, allcfg, guess):
-        # remove 'frappy_changed()' commands in script queue
-        controller = session.daemon_device._controller
-        controller.block_requests(r['reqid'] for r in controller.get_queue() if r['script'] == 'frappy_changed()')
-        self._previous_shown = allcfg, guess
-        session.log.info(all_info(allcfg))
-        if guess:
-            info = all_info(guess, '')
-            session.log.warning('please consider to call:')
-            session.log.info(info)
-            if '?' in info:
-                session.log.warning("but create cfg files first for items marked with '?'")
+    def __call__(self, *args, main=None, stick=None, addons=None, force=False, update=True):
+        """(re)start frappy server(s) with given configs and load setup if needed
 
-    def changed(self):
-        current = self.check_or_start()
-        #if current == self._previous_shown:
-        #    return
-        self.show_config(*current)
+        - without argument: list running frappy servers, restart failed frappy servers
+        - frappy('<cfg>'): if available, the standard stick is added too
+        - frappy(''): the stick is removed too
+        - addons are not changed when not given
+        - frappy(main='<cfg>')      # main cfg is changed, but stick is kept
+        - frappy('restart')         # restart all frappy servers
+        - frappy('reconnect')       # reconnect to running frappy servers
+        """
+        self._back_to_normal = None  # reset 'back to normal' machanism
+        stickarg = stick
+
+        need_change, changes, fm = to_consider = self.to_consider()
+        seacfg = fm.sea.get_cfg(config.instrument, 'sea', True).split('/', 1)
+        confirmed = seacfg[0]
+        if args:
+            if main is not None:
+                raise TypeError('got multiple values for main')
+            main = args[0]
+            if len(args) == 1:  # special case: main given as single argument
+                if main == 'restart':
+                    main = self._current_cfgs.get('main')
+                    stick = self._current_cfgs.get('stick')
+                    addons = self._current_cfgs.get('addons')
+                elif main == 'reconnect':
+                    main = None
+                elif stick is None:  # auto stick
+                    if main == '':
+                        stick = ''  # remove stick with main
+                    elif (len(seacfg) < 3 or seacfg[2] == ''  # currently no stick
+                            or seacfg[1].lower() != main.lower()):  # or main has changed
+                        stickcfg = main + 'stick'
+                        if fm.is_cfg(config.instrument, 'stick', stickcfg):
+                            # if a default stick is available, start this also
+                            stick = stickcfg
+                        else:
+                            stick = ''  # remove stick when main has changed
+            else:
+                if stick is not None:
+                    raise TypeError('got multiple values for stick')
+                stick, *alist = args[1:]
+                if alist:
+                    if addons is not None:
+                        raise TypeError('got multiple values for addons')
+                    addons = ','.join(alist)
+        elif main is None and stick is None and addons is None:  # bare frappy() command
+            if update and need_change:
+                self.update()
+            else:
+                self.show(False, to_consider)
+            return
+        if confirmed and main not in (None, 'restart') and confirmed.lower() != main.lower() and not force:
+            session.log.warning('%r is plugged to the cryostat control rack', confirmed)
+            cmd = all_info({'main': main, 'stick': stickarg, 'addons': addons}, '')[:-1] + ', force=True)'
+            session.log.warning(f'if you are sure, use: %s', cmd)
+            raise TypeError('refuse to override plugged device')
+        session.log.info(all_info(self.start_services(main, stick, addons)))
+
+    def show(self, changes_only=False, consider_result=None):
+        need_changes, changes, fm = consider_result or self.to_consider()
+        if need_changes:
+            session.log.warning('sample environment configuration should be changed:')
+        elif changes_only:
+            return
+        else:
+            session.log.info('status of sample environment services:')
+        session.log.info('  %s', 75*'_')
+        session.log.info(' ')
+        kwargs = {}
+        frappy_equals_given = True
+        rows = [['service', 'sea', 'frappy', 'nicos', 'proposed'], [''] * 5]
+        for service in SERVICES:
+            cfg = changes.get(service)
+            frappy_cfg = fm.frappy_cfgs.get(service, '-')
+            prev = self._target_cfgs.get(service, '-')
+            if not (cfg is None or isinstance(cfg, Keep)):
+                kwargs[service] = str(cfg)
+                frappy_equals_given = False
+            elif frappy_cfg != prev:
+                frappy_equals_given = False
+            rows.append([service, fm.sea_cfgs.get(service, '-'), frappy_cfg,
+                         prev, '-' if cfg is None else cfg])
+        try:
+            wid = [max(len(v) for v in column) for column in zip(*rows)]
+        except Exception as e:
+            print(e)
+            raise
+        for row in rows:
+            if not kwargs:
+                row.pop()
+                if frappy_equals_given:
+                    row.pop()
+            session.log.info('  %s', '  '.join(v.ljust(w) for w, v in zip(wid, row)))
+        if self._rebuild_env:
+            session.log.info('  %s', self._rebuild_env)
+        if kwargs:
+            alternative = f" or {all_info(kwargs, '')}" if kwargs else ''
+            session.log.info(' ')
+            session.log.info('  use frappy()%s to configure sample environment', alternative)
+        session.log.info('  %s', 75*'_')
+
+    def update(self, main=None, stick=None, addons=None):
+        self._back_to_normal = None
+        if main is None and stick is None and addons is None:
+            changes = self.to_consider()[1]
+        else:
+            changes = {k: Keep(v) for k, v in zip(SERVICES, (main, stick, addons))
+                       if v is not None}
+        session.log.info(all_info(self.start_services(**changes)))
+
+    def get_init_info(self, service):
+        """check whether a connect of this service is required"""
+        if self._initial_info is None:
+            # we do this only once for all services
+            fm = FrappyManager()
+            running = fm.get_cfg(config.instrument, None)
+            cache = self._getCache()
+            cfgs = {}
+            for serv, secnode in zip(fm.services, self.nodes):
+                if cache:
+                    cfg = cache.get(secnode, 'previous_config', '')
+                    if cfg:
+                        cfgs[serv] = cfg
+            self._initial_info = {s: (cfgs.get(s, '-'), running.get(s)) for s in fm.services}
+            fm.get_server_state(config.instrument, cfgs)
+        return self._initial_info[service]
 
     def remove_aliases(self):
         for meaning in self.meanings:
@@ -278,16 +527,15 @@ class FrappyConfig(Device):
                     result[aliasname] = aliasdev
         return result
 
-    def set_envlist(self):
-        """create aliases for SECoP devices
+    def needed_envalias(self):
+        """create aliases and envlist for SECoP devices
 
         depending on their meaning
         """
-        previous_aliases = self.get_se_aliases()
-        # self.remove_aliases()
-
         nodedevs = filter(None, [session.devices.get(devname) for devname in self.nodes])
-        sample_devices = {}
+        # value to add to the importance given in meanings targets list
+        add_importance = {k: 10 * i for i, k in enumerate(self.nodes)}
+        sample_devices = {k: [] for k in self.meanings}
         for nodedev in nodedevs:
             secnode = nodedev._secnode
             if not secnode:
@@ -300,10 +548,25 @@ class FrappyConfig(Device):
                     meaning = None
                 if meaning:
                     meaning_name, importance = meaning
-                    sample_devices.setdefault(meaning_name, []).append((importance, devname))
+                    targetlist = sample_devices.get(meaning_name)
+                    if targetlist is None:
+                        session.log.warning('%s: meaning %r is unknown', devname, meaning_name)
+                        continue
+                    sample_devices[meaning_name].append((importance, devname))
+                    head, _, tail = meaning_name.partition('_')
+                    if tail == 'regulation':
+                        # add temperature_regulation to temperature list, with very low importance
+                        sample_devices[head].append((importance - 100, devname))
+                    elif not tail:
+                        reg = f'{meaning_name}_regulation'
+                        if reg in sample_devices and isinstance(session.devices.get(devname), Moveable):
+                            sample_devices[reg].append((importance, devname))
 
+        previous_aliases = self.get_se_aliases()
         newenv = {}  # to be added to envlist (dict [devname] of aliasname)
-        to_remove = set()  # items to be removed from previous envlist, if present
+        to_remove = set()  # items to be removed or replaced, if present
+        needed_aliases = {}
+        predef_aliases = []
         for meaning in self.meanings:
             info = getattr(self, meaning)
             aliasnames = info.get('alias')
@@ -311,7 +574,7 @@ class FrappyConfig(Device):
                 aliasnames = []
             elif isinstance(aliasnames, str):
                 aliasnames = [aliasnames]
-            aliascfg = info.get('targets', {})
+            aliascfg = {k: v - i * 0.01 for i, (k, v) in enumerate(info.get('targets', {}).items())}
             predefined_alias = info.get('predefined_alias')
             if predefined_alias:
                 aliases = [a for a in predefined_alias
@@ -319,31 +582,29 @@ class FrappyConfig(Device):
                 if aliases:
                     if len(aliases) > 1:
                         raise TypeError(f'do know to which of {aliases} {meaning} to assign to')
-                    alias_config = session.alias_config.setdefault(aliases[0], [])
-                    alias_config.extend(list(aliascfg.items()))
+                    predef_aliases.append((aliases[0], list(aliascfg.items())))
             elif not aliasnames:
                 session.log.warn("neither 'predefined_alias' nor 'alias' configured. skip %s", meaning)
                 continue
             importance_list = sample_devices.get(meaning, [])
-            importance_list.extend([(nr, nam) for nam, nr in aliascfg.items() if nam in session.devices])
+            for devname, nr in aliascfg.items():
+                dev = session.devices.get(devname)
+                if dev:
+                    # increase importance by 10 for stick or 20 for addons, in case the device is
+                    # living on the stick/addons node
+                    nr += add_importance.get(str(getattr(dev, 'secnode', '')), 0)
+                    importance_list.append((nr, devname))
             importance_list = sorted(importance_list, reverse=True)
             session.log.debug('%s: %r', meaning, importance_list)
             for _, devname, in importance_list:
                 dev = session.devices.get(devname)
                 if dev is None or info.get('drivable_only', False) and not isinstance(dev, Moveable):
                     continue
+                # determine aliases
                 for aliasname in aliasnames:
-                    devcfg = ('nicos.core.DeviceAlias', {})
-                    session.configured_devices[aliasname] = devcfg
-                    session.dynamic_devices[aliasname] = 'frappy'  # assign to frappy setup
-                    aliasdev = previous_aliases.pop(aliasname, None)
-                    if aliasdev:
-                        if aliasdev.alias != devname:
-                            session.log.debug('change alias %r -> %r', aliasname, devname)
-                    else:
-                        session.log.debug('create alias %r -> %r', aliasname, devname)
-                        aliasdev = session.createDevice(aliasname, recreate=True, explicit=True)
-                    aliasdev.alias = devname
+                    if aliasname not in needed_aliases:
+                        needed_aliases[aliasname] = devname
+                # determine envlist
                 if aliasnames:
                     # only the first item of aliasnames is added to the envlist
                     aliasname = aliasnames[0]
@@ -355,23 +616,96 @@ class FrappyConfig(Device):
                         newenv[devname] = aliasname
                 break
             else:
-                to_remove.union(aliasnames)
+                to_remove.update(aliasnames)
 
-        for aliasname in previous_aliases:
-            session.destroyDevice(aliasname)
-            session.configured_devices.pop(aliasname, None)
-            session.dynamic_devices.pop(aliasname, None)
+        # determine aliases to be changed
+        for aliasname, dev in previous_aliases.items():
+            target = needed_aliases.get(aliasname)
+            if target:
+                if dev.alias == target:
+                    needed_aliases.pop(aliasname)
+            else:
+                needed_aliases[aliasname] = None
 
-        applyAliasConfig()  # for other aliases
+        # build new env list. keep order as before
+        addedenv = [v for v in newenv.values()]
+        to_remove = to_remove.difference(addedenv)
+        prevenv = [k for k in session.experiment.envlist if k not in to_remove]
+        envlist = prevenv + [k for k in addedenv if k not in prevenv]
 
-        envlist = [k for k in session.experiment.envlist if k not in to_remove] + list(newenv.values())
-        if envlist != session.experiment.envlist:
-            removed = set(session.experiment.envlist).difference(envlist)
-            session.experiment.setEnvironment(envlist)
-            if removed:
-                session.log.info('removed %s from environment', ', '.join(removed))
-            if newenv:
-                session.log.info('added %s to environment', ', '.join(newenv.values()))
+        predef_changes = []
+        for aliasname, cfg in predef_aliases:
+            if cfg and cfg[0] not in session.alias_config.get(aliasname, []):
+                predef_changes.append((aliasname, cfg))
+
+        return envlist, needed_aliases, predef_changes
+
+    def check_envalias(self):
+        envlist, new_aliases, predef_aliases = self.needed_envalias()
+        if set(envlist) != set(session.experiment.envlist):
+            return f"envlist should be {', '.join(envlist)}"
+        anew = [k for k, v in new_aliases.items() if v is not None]
+        removed = set(new_aliases).difference(anew)
+        anew.extend([k for k, _ in predef_aliases])
+        if anew:
+            return f"aliases {', '.join(anew)} should change"
+        if removed:
+            return f"aliases {', '.join(anew)} should be removed"
+        return None
+
+    def set_envalias(self):
+        """create aliases and envlist for SECoP devices
+
+        depending on their meaning (and name)
+        """
+        envlist, new_aliases, predef_aliases = self.needed_envalias()
+        if new_aliases or predef_aliases:
+            for aliasname, devname in new_aliases.items():
+                if devname is None:
+                    session.destroyDevice(aliasname)
+                    session.configured_devices.pop(aliasname, None)
+                    session.dynamic_devices.pop(aliasname, None)
+                else:
+                    dev = session.devices.get(devname)
+                    devcfg = ('nicos.core.DeviceAlias', {})
+                    session.configured_devices[aliasname] = devcfg
+                    session.dynamic_devices[aliasname] = 'frappy'  # assign to frappy setup
+                    adev = session.devices.get(aliasname)
+                    if adev:
+                        session.log.debug('change alias %r -> %r', aliasname, devname)
+                    else:
+                        session.log.debug('create alias %r -> %r', aliasname, devname)
+                        session.cache.put(aliasname, 'visibility', dev.visibility)
+                        session.cache.put(aliasname, 'loglevel', dev.loglevel)
+                        session.cache.put(aliasname, 'description', dev.description)
+                        adev = session.createDevice(aliasname, recreate=True, explicit=True)
+                    adev.alias = devname
+                    # make sure device panel is updated
+                    try:
+                        session.cache.put(devname, 'status', dev.status())
+                        session.cache.put(devname, 'value', dev.read())
+                    except Exception:
+                        pass
+            for aliasname, cfg in predef_aliases:
+                alias_config = session.alias_config.setdefault(aliasname, [])
+                if cfg not in alias_config:
+                     alias_config.extend(cfg)
+
+            applyAliasConfig()  # for other aliases
+
+        prev = set(session.experiment.envlist)
+        remove = ', '.join(prev.difference(envlist))
+        session.experiment.setEnvironment(envlist)
+        show = []
+        keep = ', '.join(d for d in envlist if d in prev)
+        if keep:
+            show.append(f'keep {keep}')
+        add = ', '.join(d for d in envlist if d not in prev)
+        if add:
+            show.append(f'add {add}')
+        if remove:
+            show.append(f'remove {remove}')
+        session.log.info('environment: %s', '; '.join(show))
 
 
 class FrappyNode(SecNodeDevice, Moveable):
@@ -389,33 +723,39 @@ class FrappyNode(SecNodeDevice, Moveable):
         'param_category': Param("category of parameters\n\n"
                                 "set to 'general' if all parameters should appear in the datafile header",
                                 type=str, default='', settable=True),
+        'quiet_init': Param('flag to set loglevel to error while initializing',
+                            type=bool, default=True, settable=True)
     }
 
     _cfgvalue = None
     _lastcfg = None
+    __log_recording = ()
 
     def doStart(self, value):
-        if value == 'None':
+        if value in ('-', 'None'):
             value = None
-        self.restart(value, True)  # frappy server will be restarted even when unchanged
-
-    def doStop(self):
-        """never busy"""
+        self.restart(value)
 
     def doInit(self, mode):
         if mode != SIMULATION and session.sessiontype != POLLER:
-            self._lastcfg = self.doRead()
-            self.restart(self._lastcfg, False)  # do not restart when not changed
+            fc = session.devices.get('frappy')
+            if fc:
+                cfg, running = fc.get_init_info(self.service)
+                self._cfgvalue = running
+                self._setROParam('target', cfg)
+                if cfg and (':' not in cfg and cfg != running):
+                    self._set_status(status.ERROR, 'cfg changed')
+                    return
         super().doInit(mode)
 
+    def doStop(self):
+        """never busy"""
+
     def doRead(self, maxage=0):
         return self._cfgvalue or ''
 
     def createDevices(self):
-        cfg = self.read()
         super().createDevices()
-        if cfg != self._lastcfg:
-            session.devices.get('frappy_config')._trigger_change.set()
         if self.param_category:
             for devname, (_, devcfg) in self.setup_info.items():
                 params_cfg = devcfg['params_cfg']
@@ -425,22 +765,58 @@ class FrappyNode(SecNodeDevice, Moveable):
                     if not pinfo.category:
                         pinfo.category = self.param_category
 
+    def makeDynamicDevices(self, setup_info):
+        patched_loggers = {}
+        if self.quiet_init:
+            for devname, (_, devcfg) in setup_info.items():
+                log = session.getLogger(devname)
+                if log not in patched_loggers:
+                    result = [loggers.INFO]  # default level
+                    patched_loggers[log] = result
+                    log.setLevel(loggers.ERROR)
+                    # avoid level change when the loglevel parameter is treated
+                    # store level instead in result
+                    log.__dict__['setLevel'] = result.append
+        try:
+            super().makeDynamicDevices(setup_info)
+        finally:
+            for log, result in patched_loggers.items():
+                log.__dict__.pop('setLevel', None)  # re-enable setLevel
+                log.setLevel(result[-1])  # set to stored or default value
+
+    def showInitLog(self):
+        for devname, record in self.__log_recording:
+            session.getLogger(devname).handle(record)
+        self.__log_recording = ()
+
     def nodeStateChange(self, online, state):
-        super().nodeStateChange(online, state)
         if online:
+            super().nodeStateChange(online, state)
             if self._cfgvalue is None:
-                self._cfgvalue = FrappyManager().get_cfg(config.instrument, self.service)
-                if not self._cfgvalue:
+                running_cfg = FrappyManager().get_cfg(config.instrument, self.service)
+                if running_cfg:
+                    fc = session.devices.get('frappy')
+                    if running_cfg != self.target and fc and not fc._restarting:
+                        self.log.warning(f'server info {running_cfg!r} does not match target cfg {self.target!r}')
+                    self._cfgvalue = running_cfg
+                else:
                     self._cfgvalue = self.uri
         else:
-            self._cfgvalue = None
-            cfg = self.read()
-            if self._lastcfg != cfg:
-                self._lastcfg = cfg
-                try:
-                    session.devices.get('frappy_config')._trigger_change.set()
-                except AttributeError:  # frappy_config does not exist
-                    pass
+            if self.target == self._cfgvalue:
+                # SecNodeDevice.nodeStateChange will change status to 'reconnecting'
+                super().nodeStateChange(online, state)
+            else:
+                # do not reconnect
+                self._cfgvalue = None
+
+    def descriptiveDataChange(self, module, description):
+        running_cfg = FrappyManager().get_cfg(config.instrument, self.service)
+        if not running_cfg or running_cfg == self.target:
+            super().descriptiveDataChange(module, description)
+        else:
+            self._disconnect(keeptarget=True)
+            self._cfgvalue = running_cfg
+            self._set_status(status.ERROR, 'cfg changed')
 
     def disable(self):
         seaconn = session.devices.get('sea_%s' % self.service)
@@ -453,76 +829,76 @@ class FrappyNode(SecNodeDevice, Moveable):
             code, text = status.DISABLED, 'disabled'
         SecNodeDevice._set_status(self, code, text)
 
-    def restart(self, cfg=None, forced=True):
+    def restart(self, cfg=None):
         """restart frappy server
 
         :param cfg: config for frappy server, if not given, restart with the same config
-        :param forced: True: restart anyway, force using cfg
-           False: try to get cfg (1) from sea, (2) from running frappy server, (3) from given cfg
-                  when cfg has not changed, do not restart
         """
         if cfg is None:
             cfg = self._cfgvalue
+            if cfg is None:
+                self.log.error('can not restart - previous cfg unknown')
+                return
         ins = config.instrument
         fm = FrappyManager()
         info = fm.get_ins_info(ins)
         running_cfg = fm.get_cfg(ins, self.service) or ''
-        if not forced or cfg is None:
-            sea_cfg = fm.cfg_from_sea(ins).get(self.service, '')
-            if '?' in sea_cfg:
-                if sea_cfg == '?':
-                    self.log.warning('undefined sea device')
-                else:
-                    self.log.warning(f"missing frappy cfg file for {sea_cfg.replace('?', '')}")
-                cfg = ''  # stop server
-            elif sea_cfg:
-                cfg = sea_cfg
-            elif running_cfg:
-                self._cfgvalue = running_cfg
-                return
-            if cfg == running_cfg:
-                self._cfgvalue = running_cfg
-                return
-        if cfg is None:
-            if forced:
-                self.log.error('can not restart - previous cfg unknown')
-            return
         if cfg != running_cfg:
             self.disable()
         if running_cfg:
-            self._disconnect()
+            self._disconnect(keeptarget=not cfg)
+            session.log.info('stop frappy_%s %r %r', self.service, running_cfg, cfg)
             fm.do_stop(ins, self.service)
-        is_cfg = cfg and ':' not in cfg
-        if is_cfg:
-            available_cfg = FrappyManager().all_cfg(config.instrument, self.service)
-            failed = False
-            for cfgitem in cfg.split(','):
-                if cfgitem not in available_cfg:
-                    failed = True
-                    suggestions = suggest(cfgitem, available_cfg)
-                    if suggestions:
-                        session.log.error('%s unknown, did you mean: %s' % (cfgitem, ', '.join(suggestions)))
-            if failed:
-                raise ValueError('use "frappy_list()" to get a list of valid frappy configurations')
-            uri = 'localhost:%d' % info[self.service]
-        else:
-            uri = cfg
-        if uri != self.uri:
-            self.uri = ''  # disconnect
-        if uri:
-            if is_cfg:
-                fm.do_start(ins, self.service, cfg, logger=self.log)
-            self.uri = uri  # connect
-        self._cfgvalue = cfg
-        if self._cache:
-            self._cache.put(self, 'value', cfg)
         self._setROParam('target', cfg)
+        try:
+            is_cfg = cfg and ':' not in cfg
+            if is_cfg:
+                if cfg == 'reconnect':
+                    is_cfg = False
+                else:
+                    available_cfg = None
+                    for cfgitem in cfg.split(','):
+                        if not fm.is_cfg(config.instrument, self.service, cfgitem):
+                            if available_cfg is None:
+                                available_cfg = fm.all_cfg(config.instrument, self.service)
+                            suggestions = suggest(cfgitem, available_cfg)
+                            if suggestions:
+                                session.log.error('%s unknown, did you mean: %s' % (cfgitem, ', '.join(suggestions)))
+                    if available_cfg is not None:
+                        raise ValueError('use "frappy_list()" to get a list of valid frappy configurations')
+                uri = 'localhost:%d' % info[self.service]
+                send_other_udp(uri, config.instrument, device=cfg)
+            else:
+                uri = cfg
+                send_other_udp(uri, config.instrument)
+            if uri != self.uri:
+                self._disconnect(keeptarget=True)
+            if uri:
+                if is_cfg:
+                    session.log.info('start frappy_%s', self.service)
+                    fm.do_start(ins, self.service, cfg, logger=self.log)
+            self.uri = uri  # connect (or disconnect)
+            self._cfgvalue = cfg
+            if self._cache:
+                self._cache.put(self, 'value', cfg)
+        finally:
+            self._cache.put(self, 'previous_config', self._cfgvalue or self.uri)
+
+    def _disconnect(self, keeptarget=False):
+        super()._disconnect()
+        if not keeptarget:
+            self._setROParam('target', '')
 
     def get_info(self):
         result = self.doRead() or ''
         code, text = self.status()
+        if not result and self.target not in ('', '-'):
+            return '<disconnected>'
         if code == status.OK or result == '':
             return result
         if (code, text) == (status.ERROR, 'reconnecting'):
             return '%s (frappy not running)' % result
         return '%s (%s)' % (result, text)
+
+    def doFinish(self):
+        return False  # avoid warning in finish() when target does not match

labcommands.py

@@ -0,0 +1,494 @@
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Markus Zolliker <markus.zolliker@psi.ch>
+#
+# *****************************************************************************
+"""collection of commands and devices for lab experiments
+
+the commands are probably better not used, as the standard scan and sweep
+commands are overwritten - this might be confusing for users working also
+in neutron instruments
+"""
+
+import os
+from os.path import join
+import inspect
+import shutil
+import re
+from glob import glob
+import time
+import math
+
+from nicos import session
+from nicos.core import Override, Param, Measurable, Moveable, Device, \
+    status, Readable
+from nicos.core.errors import NicosError
+from nicos.commands import helparglist, usercommand, parallel_safe
+from nicos.commands.device import maw
+import nicos.devices.generic.paramdev as paramdev
+import nicos.commands.scan
+from nicos_sinq.frappy_sinq.timestamp import Timestamp, Det
+
+
+class FunDevice(Moveable):
+    """wrap a device in a function
+
+    see usercommand fun
+    """
+    temporary = True
+    _dev = None
+    _value = None
+
+    def __init__(self, func, dev_or_name=None, unit=None, devname=None, **kwargs):
+        self._func = func
+        self._kwargs = kwargs
+        if isinstance(dev_or_name, Device):
+            self._dev = dev_or_name
+            self._devname = dev_or_name.name
+            if devname:
+                # or should we raise an error?
+                name = devname
+            else:
+                name = self._devname
+        else:
+            name = dev_or_name or devname
+            self._devname = devname
+            self._dev = session.devices.get(devname)
+        if self._dev and unit is None:
+            unit = self._dev.unit
+        Moveable.__init__(self, name, unit=unit or '')
+
+    def doRead(self, maxage=0):
+        if self._devname:
+            if not self._dev:
+                try:
+                    self._dev = session.devices[self._devname]
+                    if not self.unit:
+                        self.unit = self._dev.unit
+                except KeyError:
+                    raise NicosError('device %r does not exist' % self._devname)
+                if self._value is None:
+                    self._value = self._dev.read(maxage)
+        return self._value
+
+    def doStart(self, value):
+        if self._devname:
+            oldvalue = self.doRead()
+        else:
+            oldvalue = self._value
+        # append args depending in function signature
+        argnames, varargs, varkw = inspect.getfullargspec(self._func)[0:3]
+        args = self._dev, oldvalue
+        kwds = dict(self._kwargs)
+        if 'oldvalue' in argnames:
+            kwds['oldvalue'] = oldvalue
+            args = args[:1]
+        if 'dev' in argnames:
+            kwds['dev'] = self._dev
+            args = ()
+        if not varkw:
+            for k in kwds:
+                if k not in argnames:
+                    kwds.pop(k)
+        result = self._func(value, *args[:len(argnames) - 1], **kwds)
+        self._value = value if result is None else result
+
+    def doIsCompleted(self):
+        return True
+
+    def doStop(self):
+        if self._dev:
+            self._dev.stop()
+
+
+@usercommand
+@helparglist('dev_or_name[, unit[, devname=..., **kwargs]]')
+def fun(dev_or_name=None, unit=None, devname=None, **kwargs):
+    """decorator for turning a function into a device for scans
+
+    Usage:
+
+    A function wrapped around a device (the argument might be a device name or a device)
+
+    @fun(dev="mydevice"):
+    def func(value, dev):
+        do_something_before_move()
+        maw(dev, value)
+        do_something_after_move()
+
+    scan(func, ...)
+
+    func will inherit the name and unit from the device.
+    func may have 1-3 positional arguments:
+    - value: the value to move to
+    - dev: the device given (in above example mydevice)
+    - oldvalue: the value before the move
+
+    A function turned into a device:
+
+    @fun("myname", "K")
+    def func(value):
+        result = do_some_thing(value)
+        return result
+
+    if no name is given, the function name is used.
+    in this case the final result may be returned, if different from value
+    """
+    def decorator(func, name=dev_or_name):
+        if name is None:
+            name = func.__name__
+        return FunDevice(func, name, unit, devname, **kwargs)
+
+    return decorator
+
+
+class Range(tuple):
+    def __new__(cls, astext, *args):
+        return tuple.__new__(cls, args)
+
+    def __init__(self, astext, *args):
+        super().__init__()
+        self.astext = astext
+
+    def __repr__(self):
+        return self.astext
+
+    def __add__(self, other):
+        astext = '%s+%s' % (repr(self), repr(other))
+        try:
+            if self[-1] == other[0]:
+                other = other[1:]
+        except IndexError:
+            pass
+        return Range(astext, *self, *other)
+
+
+@usercommand
+@helparglist('start, step, end')
+def lnr(start, step_or_end=None, end=None, n=None):
+    """linear range
+
+    to be used as argument for the scan command
+
+    Alternative form:
+
+    >>> lnr(start, end, n=n)  # n+1 points
+
+    ranges might be added, adjacent equal points are omitted automatically
+
+    >>> list(lnr(0,1,4) + lnr(4,2,10) + [15])
+    > [0,1,2,3,4,6,8,10,15]
+    """
+    if end is None:
+        astext = 'lnr(%s,%s,n=%s)' % (start, step_or_end, n)
+        if step_or_end is None:
+            # single value
+            return Range('lnr(%s)' % start, start)
+        end = step_or_end
+        if (n or 1) == 1:
+            # two values
+            return Range('lnr(%s,%s)' % (start, end), start, end)
+        n = abs(n)
+        step = (end - start) / n
+    else:
+        step = step_or_end
+        if n is not None or step == 0:
+            raise ValueError('illegal arguments')
+        astext = 'lnr(%s,%s,%s)' % (start, step_or_end, end)
+        n = int(round(abs((end - start) / step)))
+        if (start < end) == (step < 0):
+            step = -step
+    return Range(astext, start, *tuple(start + i * step for i in range(1,n)), end)
+
+
+@usercommand
+@helparglist('start, factor, end')
+def lgr(start, factor_or_end, end=None, n=0):
+    """logarithmic range
+
+    factor is the factor between two points (0.5 and 2 are equivalent)
+
+    Alternative form:
+
+    >>> lgr(start, end, n=n)  # n+1 points
+
+    ranges might be added, adjacent equal points are omitted automatically
+
+    list(lnr(0,1,4) + lgr(4,2,15) + [30]) == [0,1,2,3,4,8,15,30]
+    """
+    if end is None:
+        end = factor_or_end
+        if start <= 0 or end <= 0 or n <= 0:
+            raise ValueError('illegal arguments')
+        astext = 'lgr(%s,%s,n=%s)' % (start, end, n)
+        factor = (end / start) ** (1 / n)
+    else:
+        factor = factor_or_end
+        if start <= 0 or end <= 0 or factor <= 0:
+            raise ValueError('illegal arguments')
+        if (start > end) == (factor > 1):
+            factor = 1 / factor
+        astext = 'lgr(%s,%.18g,%s)' % (start, factor, end)
+    n = int(round(abs(math.log2(end / start) / math.log2(factor))))
+    return Range(astext, start,
+                 *(start * factor ** i for i in range(1, n)),
+                 end)
+
+
+class WithTimeoutDev(Moveable):
+    """Wrapper for better timeout/settling mechanism"""
+
+    parameters = {
+        'dev': Param('dev', type=lambda x=None: x, settable=True, mandatory=True),
+        'tolerance': Param('tolerance', type=float,
+                           settable=True, default=1, mandatory=False),
+        'settle': Param('time to settle within tolerance', type=float,
+                        settable=True, default=1, mandatory=False),
+        'timeout': Param('timeout for less progress than tolerance (defaults to settle)', type=float,
+                         settable=True, default=-1, mandatory=False),
+    }
+
+    temporary = True
+    _start_time = None
+
+    def _getCache(self):
+        return None
+
+    def doStart(self, value):
+        self._start_time = time.time()
+        self._settle_start = None
+        self._settle_done = 0
+        self._mindif = abs(value - self.dev())
+        self.dev.doStart(value)
+
+    def doStop(self):
+        self._start_time = None
+        self.dev.doStop()
+
+    def doRead(self, maxage=0):
+        return self.dev.doRead(maxage)
+
+    def doStatus(self, maxage=0):
+        if not self._start_time:
+            return status.OK, ''
+        now = time.time()
+        dif = abs(self.doRead(maxage) - self.target)
+        if dif < self.tolerance:
+            if not self._settle_start:
+                self.log.info('settling %s' % self.dev)
+                self._settle_start = now - self._settle_done
+            if now > self._settle_start + self.settle:
+                self._start_time = None
+                return status.OK, ''
+        else:
+            if now > self._start_time + (self.timeout or self.settle):
+                self._start_time = None
+                self.log.info('timeout waiting for %s' % self.dev)
+                return status.OK, ''
+            if self._settle_start:
+                self._settle_done = now - self._settle_start
+                self._settle_start = None
+        if dif < self._mindif:
+            self._mindif = dif - self.tolerance
+            self._start_time = now
+        return status.BUSY, ''
+
+
+@usercommand
+@helparglist('dev, tolerance [,settle [,timeout]]')
+@parallel_safe
+def WithTimeout(dev, tolerance, settle=300, timeout=0):
+    if isinstance(dev, WithTimeoutDev):
+        dev.tolerance = tolerance
+        dev.settle = settle
+        dev.timeout = timeout
+        return dev
+    return WithTimeoutDev(dev.name, dev=dev, tolerance=tolerance, settle=settle, timeout=timeout,
+                          unit=dev.unit, fmtstr=dev.fmtstr)
+
+
+@usercommand
+@helparglist('dev, target[, tolerance[,settle[, timeout]]]')
+def maw_tmo(dev, target, tolerance, settle=300, timeout=0):
+    """wait for target reached or no progress within timeout
+
+    wait for either target within tolerance for a total of <settle> seconds
+    or no progress more than tolerance for <timeout> seconds
+    """
+    maw(WithTimeout(dev, tolerance, settle, timeout), target)
+
+
+class ReadonlyParamDevice(paramdev.ReadonlyParamDevice):
+    """turn a parameter into a temporary Readable"""
+    temporary = True
+
+    def __init__(self, devname_param):
+        devname, pname = devname_param.split('.')
+        paramdev.ReadonlyParamDevice.__init__(self, devname_param, device=devname, parameter=pname)
+
+    def _getCache(self):
+        return None
+
+
+class ParamDevice(paramdev.ParamDevice):
+    """turn a parameter into a temporary Moveable"""
+    temporary = True
+
+    def __init__(self, devname_param):
+        devname, pname = devname_param.split('.')
+        paramdev.ParamDevice.__init__(self, devname_param, device=devname, parameter=pname)
+
+    def _getCache(self):
+        return None
+
+
+@usercommand
+@helparglist('"dev.param, dev.param, ..."')
+@parallel_safe
+def out(*args):
+    """may be used as an argument in a scan for producing output columns for parameters
+
+    can not be used in SetEnvironment
+
+    For a single parameter:
+       scan(dev, ...., out('tt.raw'))
+
+    For multiple parameters:
+       scan(dev, ...., *out('tt, tt.raw, mf.ramp'))
+    """
+    result = []
+    for arg in args:
+        for devpar in arg.split(','):
+            devpar = devpar.strip()
+            if '.' in devpar:
+                result.append(ReadonlyParamDevice(devpar))
+            else:
+                result.append(session.devices[devpar])
+    if len(result) == 1:
+        return result[0]
+    return result
+
+
+@usercommand
+@helparglist('"dev.param"')
+@parallel_safe
+def param(devpar):
+    """turning a parameter into a moveable device
+
+    Usage:
+
+        scan(param(dev), ....)
+    """
+    return ParamDevice(devpar)
+
+
+def copy_all(srcdir, dstdir):
+    """copy all files from srcdir to dstdir"""
+    files = glob(join(srcdir, '*'))
+    for src in files:
+        shutil.copy2(src, dstdir)
+    return files
+
+
+@usercommand
+@helparglist('script path | instrument, proposal')
+@parallel_safe
+def copy_scripts(ins_or_dir, proposal=None):
+    """copy scripts from another proposal or any directory"""
+    if os.path.isdir(ins_or_dir):
+        dirname = ins_or_dir
+    else:
+        data = join(os.environ['NICOS_DATA'], ins_or_dir)
+        if isinstance(proposal, int):
+            pat = re.compile('.*/(.*\D|)0*%d$' % proposal)
+        else:
+            pat = re.compile('.*/%s$' % re.escape(proposal))
+        for dirname in reversed(sorted(glob(join(data, '20*', '*')))):
+            if pat.match(dirname):
+                break
+        else:
+            raise FileNotFoundError('directory for %s/%s not found' % (ins_or_dir, proposal))
+    copy_all(join(dirname, 'scripts'), join(session.devices['Exp'].scriptpath))
+
+
+def handle_args(args):
+    detectors = []
+    put_timestamp = False
+    for det in session.experiment.detectors:
+        if isinstance(det, Timestamp) and det.show:
+            put_timestamp = True
+    ret = []
+    moveable_seen = False
+    for arg in args:
+        if moveable_seen and isinstance(arg, Readable):
+            if put_timestamp:
+                 ret.append(TimestampReadable())
+                 put_timestamp = False
+            #ret.append(arg)
+            ret.append(Det(arg))
+        elif isinstance(arg, str) and '.' in arg:
+            ret.append(param(arg))
+            moveable_seen = True
+        elif isinstance(arg, Measurable):
+            if arg not in session.experiment.detectors:
+                ret.append(arg)
+        else:
+            if isinstance(arg, Moveable):
+                moveable_seen = True
+            ret.append(arg)
+    return ret
+
+
+@usercommand
+def scan(*args, **kwargs):
+    nicos.commands.scan.scan(*handle_args(args), **kwargs)
+
+
+scan.help_arglist = nicos.commands.scan.scan.help_arglist
+scan.__doc__ = nicos.commands.scan.scan.__doc__
+
+
+@usercommand
+def sweep(dev, start, end, *args, **kwargs):
+    nicos.commands.scan.sweep(dev, start, end, *handle_args(args), **kwargs)
+
+
+sweep.help_arglist = nicos.commands.scan.sweep.help_arglist
+sweep.__doc__ = nicos.commands.scan.sweep.__doc__
+
+
+class TimestampReadable(Readable):
+    """timestamp as Readable"""
+    temporary = True
+
+    def __init__(self):
+        Readable.__init__(self, 'timestamp', unit='s')
+
+    def _getCache(self):
+        """no cache needed"""
+        self._cache = None
+
+    def doRead(self, maxage=0):
+        return time.time()
+
+    def doStatus(self, maxage=0):
+        return 100, ''
+
+    def valueInfo(self):
+        return Readable.valueInfo(self)
+

setups/frappy.py

@@ -4,10 +4,8 @@ group = 'optional'
 modules = ['nicos_sinq.frappy_sinq.commands']
 
 devices = dict(
-    frappy_config = device(
+    frappy = device(
         'nicos_sinq.frappy_sinq.devices.FrappyConfig',
-        # frappy_config does not need to be visible
-        visibility = [],
         # the possible SECoP connections
         nodes = ['se_main', 'se_stick', 'se_addons'],
         #
@@ -19,18 +17,19 @@ devices = dict(
         # the devices with the names given by key are added to determine the
         # device, using the given importance number, with similar values as
         # given by the SECoP standard (10: instrument, 20: cryostat, 30: insert)
+        # the given numbers assume it is on the cryo, +10 / +20 is added for stick/addons
         temperature = {  # the SECoP meaning
-            'alias': ['Ts', 'temperature'],  # the name(s) to be given to the alias
-            'targets':  # possible devices in addition with importance
-               {'se_ts': 20, 'se_tt': 19, 'se_tm': 18},
+            'alias': 'Ts',  # the name(s) to be given to the alias
+            'targets':  # possible devices in addition with importance (numbers assume its on the cryo)
+               {'se_ts': 20, 'se_tt': 20, 'se_tm': 20},
         },
         temperature_regulation = {
             'alias': 'T',
-            'targets': {'se_ts': 20, 'se_tt': 19, 'se_tm': 18, 'se_T_stat': 17},
+            'targets': {'se_ts': 27, 'se_tt': 27, 'se_tm': 27, 'se_T_stat': 27},
             'drivable_only': True,
         },
         magneticfield = {
-            'alias': ['B', 'magfield'],
+            'alias': 'B',
             'targets': {'se_mf': 20},
         },
         pressure = {
@@ -46,22 +45,24 @@ devices = dict(
         },
         stick_rotation={
             'alias': 'dom',
-            'targets': {'se_om': 20, 'se_stickrot': 19},
+            'targets': {'se_om': 20, 'se_stickrot': 20},
             'envlist': False,
         }
     )
 )
 
 startupcode = '''
-printinfo("=======================================================================================")
-printinfo("Welcome to the NICOS frappy secnode setup!")
+printinfo("  ___________________________________________________________________________________________")
 printinfo(" ")
-printinfo("Usage:")
-printinfo("  frappy('<main cfg>')                 # change main SE configuration (e.g. cryostat)")
-printinfo("  frappy('<main cfg>', '<stick cfg>')  # change main and stick cfg")
-printinfo("  frappy(stick='')                     # remove stick")
-printinfo("  frappy('')                           # remove main SE apparatus")
-printinfo("  frappy()                             # show the current SE configuration")
-printinfo("=======================================================================================")
+printinfo("  Welcome to the NICOS frappy secnode setup!")
+printinfo(" ")
+printinfo("  Usage:")
+printinfo("    frappy('<main cfg>')                 # change main SE configuration (e.g. cryostat)")
+printinfo("    frappy('<main cfg>', '<stick cfg>')  # change main and stick cfg")
+printinfo("    frappy(stick='')                     # remove stick")
+printinfo("    frappy('')                           # remove main SE apparatus")
+printinfo("    frappy.read()                        # show the current SE configuration")
+printinfo("    frappy()                             # show and update SE configuration form server state")
+printinfo("  ___________________________________________________________________________________________")
 set_se_list()
 '''

setups/frappy_addons.py

@@ -6,6 +6,6 @@ devices = {
     'se_addons':
         device('nicos_sinq.frappy_sinq.devices.FrappyNode',
                description='SEC node', unit='', async_only=True,
-               prefix='se_', auto_create=True, service='addons',
+               prefix=environ.get('SE_PREFIX', 'se_'), auto_create=True, service='addons',
         ),
 }

setups/frappy_main.py

@@ -6,6 +6,6 @@ devices = {
     'se_main':
         device('nicos_sinq.frappy_sinq.devices.FrappyNode',
                description='main SEC node', unit='', async_only=True,
-               prefix='se_', auto_create=True, service='main',
+               prefix=environ.get('SE_PREFIX', 'se_'), auto_create=True, service='main',
         ),
 }

setups/frappy_stick.py

@@ -6,6 +6,6 @@ devices = {
     'se_stick':
         device('nicos_sinq.frappy_sinq.devices.FrappyNode',
                description='stick SEC node', unit='', async_only=True,
-               prefix='se_', auto_create=True, service='stick',
+               prefix=environ.get('SE_PREFIX', 'se_'), auto_create=True, service='stick',
         ),
 }

setups/ppms.py

@@ -0,0 +1,26 @@
+from os import environ
+description = 'frappy main setup'
+group = 'optional'
+
+devices = {
+    'se_main':
+        device('nicos_sinq.frappy_sinq.devices.FrappyNode',
+               uri='pc12694:5000',
+               description='main SEC node', unit='',
+               prefix=environ.get('SE_PREFIX', ''), auto_create=True, service='main',
+        ),
+    'timestamp': device('nicos_sinq.frappy_sinq.lab.Timestamp', description='time, a dummy detector'),
+}
+
+startupcode = '''
+printinfo("=======================================================================================")
+printinfo("Welcome to the NICOS frappy secnode setup for PPMS!")
+printinfo(" ")
+printinfo("Usage:")
+printinfo("  frappy(stick='<stick cfg>')                        # change sample-stick configuration")
+printinfo("  frappy(addons='<addon1 cfg>,<addon2 cfg> ...')     # change SE addons")
+printinfo("  frappy(stick=None)                                 # remove stick")
+printinfo("  frappy(addons=None)                                # remove addons")
+printinfo("=======================================================================================")
+SetDetectors(timestamp)
+'''

setups/timestamp.py

@@ -0,0 +1,9 @@
+description = 'timestamp dummy detector for offline measurements'
+group = 'optional'
+
+devices = {
+    'timestamp': device('nicos_sinq.frappy_sinq.lab.Timestamp', description='time, a dummy detector'),
+}
+
+startupcode = 'SetDetectors(timestamp)'
+

timestamp.py

@@ -0,0 +1,124 @@
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Markus Zolliker <markus.zolliker@psi.ch>
+#
+# *****************************************************************************
+"""helper devices for lab experiments"""
+
+import time
+
+from nicos.core import Override, Param, Measurable
+from nicos.core.params import Value
+
+
+class Timestamp(Measurable):
+    """to be used as a 'detector' in lab experiments
+
+    just waiting the preset time
+    """
+    parameter_overrides = {
+        'unit': Override(default='sec', mandatory=False),
+    }
+    parameters = {
+        'show': Param('show timestamp in data file', type=bool,
+                      settable=True, default=True, mandatory=False),
+    }
+
+    _preset = 0
+    _time_used = 0
+    _start = 0
+
+    def _getCache(self):
+        """no cache needed"""
+        self._cache = None
+
+    def doRead(self, maxage=0):
+        return time.time()
+
+    def doStatus(self, maxage=0):
+        return 100, ''
+
+    def doSetPreset(self, t=0, **preset):
+        self._start = 0
+        self._preset = t
+        self._value = 0
+
+    def doStart(self):
+        self._start = time.time()
+
+    def doPause(self):
+        self._time_used = time.time() - self._start
+        return True
+
+    def doResume(self):
+        self._start = time.time() - self._time_used
+
+    def doFinish(self):
+        pass
+
+    def doStop(self):
+        pass
+
+    def doIsCompleted(self):
+        return time.time() > self._start + self._preset
+
+    def valueInfo(self):
+        if self.show:
+            return Measurable.valueInfo(self)
+        return ()
+
+
+class Det(Measurable):
+    """wrap a Readable into a Detector
+
+    just for placing the result in a scan file in the detector part
+    """
+    temporary = True
+
+    def __init__(self, dev):
+        self._dev = dev
+        Measurable.__init__(self, str(dev), unit=dev.unit, fmtstr=dev.fmtstr)
+
+    def _getCache(self):
+        """no cache needed"""
+        self._cache = None
+
+    def doRead(self, maxage=0):
+        return self._dev.doRead(maxage)
+
+    def doStatus(self, maxage=0):
+        return self._dev.doStatus(maxage)
+
+    def doSetPreset(self, t=5, **preset):
+        self._start = 0
+        self._preset = t
+
+    def doStart(self):
+        self._start = time.time()
+
+    def doFinish(self):
+        pass
+
+    def doStop(self):
+        self._start = 0
+
+    def doIsCompleted(self):
+        return time.time() > self._start + self._preset
+
+    def valueInfo(self):
+        return Value(self.name, unit=self.unit, fmtstr=self.fmtstr),