generalConfig, config: use pathlib

- switch to pathlib
- represent multiple confdirs as list of Paths internally, not string
  with pathsep

Change-Id: I1418e561641e27cd904af0762be056cd66ee1919
Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/34464
Reviewed-by: Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Alexander Zaft <a.zaft@fz-juelich.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

frappy/config.py

@@ -20,6 +20,7 @@
 # *****************************************************************************
 
 import os
+from pathlib import Path
 import re
 from collections import Counter
 
@@ -128,8 +129,7 @@ class Config(dict):
 
 
 def process_file(filename, log):
-    with open(filename, 'rb') as f:
-        config_text = f.read()
+    config_text = filename.read_bytes()
     node = NodeCollector()
     mods = Collector(Mod)
     ns = {'Node': node.add, 'Mod': mods.add, 'Param': Param, 'Command': Param, 'Group': Group}
@@ -149,22 +149,21 @@ def process_file(filename, log):
 def to_config_path(cfgfile, log):
     candidates = [cfgfile + e for e in ['_cfg.py', '.py', '']]
     if os.sep in cfgfile:  # specified as full path
-        filename = cfgfile if os.path.exists(cfgfile) else None
+        file = Path(cfgfile) if Path(cfgfile).exists() else None
     else:
-        for filename in [os.path.join(d, candidate)
-                         for d in generalConfig.confdir.split(os.pathsep)
-                         for candidate in candidates]:
-            if os.path.exists(filename):
+        for file in [Path(d) / candidate
+                     for d in generalConfig.confdir
+                     for candidate in candidates]:
+            if file.exists():
                 break
         else:
-            filename = None
-
-    if filename is None:
+            file = None
+    if file is None:
         raise ConfigError(f"Couldn't find cfg file {cfgfile!r} in {generalConfig.confdir}")
-    if not filename.endswith('_cfg.py'):
-        log.warning("Config files should end in '_cfg.py': %s", os.path.basename(filename))
-    log.debug('Using config file %s for %s', filename, cfgfile)
-    return filename
+    if not file.name.endswith('_cfg.py'):
+        log.warning("Config files should end in '_cfg.py': %s", file.name)
+    log.debug('Using config file %s for %s', file, cfgfile)
+    return file
 
 
 def load_config(cfgfiles, log):

frappy/lib/__init__.py

@@ -31,6 +31,7 @@ import threading
 import traceback
 from configparser import ConfigParser
 from os import environ, path
+from pathlib import Path
 
 
 SECoP_DEFAULT_PORT = 10767
@@ -70,28 +71,32 @@ class GeneralConfig:
         """
         cfg = {}
         mandatory = 'piddir', 'logdir', 'confdir'
-        repodir = path.abspath(path.join(path.dirname(__file__), '..', '..'))
+        repodir = Path(__file__).parents[2].expanduser().resolve()
         # create default paths
-        if (path.splitext(sys.executable)[1] == ".exe"
-           and not path.basename(sys.executable).startswith('python')):
+        if (Path(sys.executable).suffix == ".exe"
+           and not Path(sys.executable).name.startswith('python')):
             # special MS windows environment
-            self.update_defaults(piddir='./', logdir='./log', confdir='./')
+            self.update_defaults(piddir=Path('./'), logdir=Path('./log'), confdir=Path('./'))
         elif path.exists(path.join(repodir, 'cfg')):
             # running from git repo
-            self.set_default('confdir', path.join(repodir, 'cfg'))
+            self.set_default('confdir', repodir / 'cfg')
             # take logdir and piddir from <repodir>/cfg/generalConfig.cfg
         else:
             # running on installed system (typically with systemd)
-            self.update_defaults(piddir='/var/run/frappy', logdir='/var/log', confdir='/etc/frappy')
+            self.update_defaults(
+                piddir=Path('/var/run/frappy'),
+                logdir=Path('/var/log'),
+                confdir=Path('/etc/frappy')
+            )
         if configfile is None:
             configfile = environ.get('FRAPPY_CONFIG_FILE')
             if configfile:
-                configfile = path.expanduser(configfile)
-                if not path.exists(configfile):
+                configfile = Path(configfile).expanduser()
+                if not configfile.exists():
                     raise FileNotFoundError(configfile)
             else:
-                configfile = path.join(self['confdir'], 'generalConfig.cfg')
-                if not path.exists(configfile):
+                configfile = self['confdir'] / 'generalConfig.cfg'
+                if not configfile.exists():
                     configfile = None
         if configfile:
             parser = ConfigParser()
@@ -100,16 +105,16 @@ class GeneralConfig:
             # only the FRAPPY section is relevant, other sections might be used by others
             for key, value in parser['FRAPPY'].items():
                 if value.startswith('./'):
-                    cfg[key] = path.abspath(path.join(repodir, value))
+                    cfg[key] = (repodir / value).absolute()
                 else:
                     # expand ~ to username, also in path lists separated with ':'
                     cfg[key] = ':'.join(path.expanduser(v) for v in value.split(':'))
             if cfg.get('confdir') is None:
-                cfg['confdir'] = path.dirname(configfile)
+                cfg['confdir'] = configfile.parent
         for key in mandatory:
             env = environ.get(f'FRAPPY_{key.upper()}')
             if env is not None:
-                cfg[key] = env
+                cfg[key] = Path(env)
         missing_keys = [
             key for key in mandatory
             if cfg.get(key) is None and self.defaults.get(key) is None
@@ -119,6 +124,8 @@ class GeneralConfig:
                 raise KeyError(f"missing value for {' and '.join(missing_keys)} in {configfile}")
             raise KeyError('missing %s'
                            % ' and '.join('FRAPPY_%s' % k.upper() for k in missing_keys))
+        if isinstance(cfg['confdir'], Path):
+            cfg['confdir'] = [cfg['confdir']]
         # this is not customizable
         cfg['basedir'] = repodir
         self._config = cfg

frappy/persistent.py

@@ -75,9 +75,9 @@ class PersistentMixin(Module):
 
     def __init__(self, name, logger, cfgdict, srv):
         super().__init__(name, logger, cfgdict, srv)
-        persistentdir = os.path.join(generalConfig.logdir, 'persistent')
+        persistentdir = generalConfig.logdir / 'persistent'
         os.makedirs(persistentdir, exist_ok=True)
-        self.persistentFile = os.path.join(persistentdir, f'{self.secNode.equipment_id}.{self.name}.json')
+        self.persistentFile = persistentdir / f'{self.secNode.equipment_id}.{self.name}.json'
         self.initData = {}  # "factory" settings
         loaded = self.loadPersistentData()
         for pname, pobj in self.parameters.items():
@@ -147,10 +147,10 @@ class PersistentMixin(Module):
                 if getattr(v, 'persistent', False)}
         if data != self.persistentData:
             self.persistentData = data
-            persistentdir = os.path.dirname(self.persistentFile)
-            tmpfile = self.persistentFile + '.tmp'
-            if not os.path.isdir(persistentdir):
-                os.makedirs(persistentdir, exist_ok=True)
+            persistentdir = self.persistentFile.parent
+            tmpfile = self.persistentFile.parent / (self.persistentFile.name + '.tmp')
+            if not persistentdir.is_dir():
+                persistentdir.mkdir(parents=True, exist_ok=True)
             try:
                 with open(tmpfile, 'w', encoding='utf-8') as f:
                     json.dump(self.persistentData, f, indent=2)

frappy/server.py

@@ -112,7 +112,7 @@ class Server:
             raise ConfigError('No interface specified in configuration or arguments!')
 
         self._cfgfiles = cfgfiles
-        self._pidfile = os.path.join(generalConfig.piddir, name + '.pid')
+        self._pidfile = generalConfig.piddir / (name + '.pid')
         signal.signal(signal.SIGINT, self.signal_handler)
         signal.signal(signal.SIGTERM, self.signal_handler)
 
@@ -127,9 +127,9 @@ class Server:
     def start(self):
         if not DaemonContext:
             raise ConfigError('can not daemonize, as python-daemon is not installed')
-        piddir = os.path.dirname(self._pidfile)
-        if not os.path.isdir(piddir):
-            os.makedirs(piddir)
+        piddir = self._pidfile.parent
+        if not piddir.is_dir():
+            piddir.mkdir(parents=True)
         pidfile = pidlockfile.TimeoutPIDLockFile(self._pidfile)
 
         if pidfile.is_locked():