refactor: add system_config and review docs

bec_lib/bec_lib/client.py

@@ -12,6 +12,7 @@ import inspect
 from typing import TYPE_CHECKING
 
 import redis
+from pydantic import BaseModel, Field, field_validator
 from rich.console import Console
 from rich.table import Table
 
@@ -39,6 +40,39 @@ ScanManager = lazy_import_from("bec_lib.scan_manager", ("ScanManager",))
 Scans = lazy_import_from("bec_lib.scans", ("Scans",))
 
 
+class SystemConfig(BaseModel):
+    """System configuration model"""
+
+    file_suffix: str | None = Field(default=None)
+    file_directory: str | None = Field(default=None)
+    model_config: dict = {"validate_assignment": True}
+
+    @field_validator("file_suffix", "file_directory")
+    @staticmethod
+    def check_validity(value: str, field: Field) -> str:
+        """Check the validity of the value
+
+        Args:
+            value (str): The value to check
+            field_name (str): The name of the field
+
+        Returns:
+            str: The value if it is valid
+        """
+        if value is None:
+            return value
+        field_name = field.field_name
+        check_value = value.replace("_", "").replace("-", "")
+        if field_name == "file_directory":
+            value = value.strip("/")
+            check_value = check_value.replace("/", "")
+        if not check_value.isalnum() or not check_value.isascii():
+            raise ValueError(
+                f"{field_name} must only contain alphanumeric ASCII characters. Provided string is: {value}"
+            )
+        return value
+
+
 class BECClient(BECService, UserScriptsMixin):
     """
     The BECClient class is the main entry point for the BEC client and all derived classes.
@@ -82,6 +116,7 @@ class BECClient(BECService, UserScriptsMixin):
         self.bl_checks = None
         self._hli_funcs = {}
         self.metadata = {}
+        self.system_config = SystemConfig()
         self.callbacks = CallbackHandler()
         self._parent = parent if parent is not None else self
         self._initialized = True

bec_lib/bec_lib/scans.py

@@ -17,6 +17,7 @@ from typeguard import typechecked
 
 from bec_lib import messages
 from bec_lib.bec_errors import ScanAbortion
+from bec_lib.client import SystemConfig
 from bec_lib.device import DeviceBase
 from bec_lib.endpoints import MessageEndpoints
 from bec_lib.logger import bec_logger
@@ -51,6 +52,8 @@ class ScanObject:
         hide_report: bool = False,
         metadata: dict = None,
         monitored: list[str | DeviceBase] = None,
+        file_suffix: str = None,
+        file_directory: str = None,
         **kwargs,
     ) -> ScanReport:
         """
@@ -76,28 +79,21 @@ class ScanObject:
         # pylint: disable=protected-access
         hide_report = hide_report or scans._hide_report
 
-        md = deepcopy(self.client.metadata)
-        key_parameter = ["sample_name", "file_suffix", "file_directory"]
-        for key in key_parameter:
-            if key not in md:
-                var = self.client.get_global_var(key)
-                if var is not None:
-                    md[key] = var
+        user_metadata = deepcopy(self.client.metadata)
+
+        sys_config = self.client.system_config.model_copy(deep=True)
+        if file_suffix:
+            sys_config.file_suffix = file_suffix
+        if file_directory:
+            sys_config.file_directory = file_directory
+
+        if "sample_name" not in user_metadata:
+            var = self.client.get_global_var("sample_name")
+            if var is not None:
+                user_metadata["sample_name"] = var
 
         if metadata is not None:
-            md.update(metadata)
-
-        if "file_suffix" in md:
-            suffix = md.get("file_suffix")
-            check_suffix = suffix.replace("_", "").replace("-", "")
-            if not check_suffix.isalnum() or not check_suffix.isascii():
-                raise ValueError("file_suffix must only contain alphanumeric ASCII characters.")
-        if "file_directory" in md:
-            directory = md.get("file_directory")
-            check_directory = directory.replace("/", "").replace("_", "").replace("-", "")
-            if not check_directory.isalnum() or not check_directory.isascii():
-                raise ValueError("file_directory must only contain alphanumeric ASCII characters.")
-            md["file_directory"] = directory.strip("/")
+            user_metadata.update(metadata)
 
         if monitored is not None:
             if not isinstance(monitored, list):
@@ -111,15 +107,17 @@ class ScanObject:
                         )
             kwargs["monitored"] = monitored
 
+        sys_config = sys_config.model_dump()
         # pylint: disable=protected-access
         if scans._scan_group:
-            md["queue_group"] = scans._scan_group
+            sys_config["queue_group"] = scans._scan_group
         if scans._scan_def_id:
-            md["scan_def_id"] = scans._scan_def_id
+            sys_config["scan_def_id"] = scans._scan_def_id
         if scans._dataset_id_on_hold:
-            md["dataset_id_on_hold"] = scans._dataset_id_on_hold
+            sys_config["dataset_id_on_hold"] = scans._dataset_id_on_hold
 
-        kwargs["metadata"] = md
+        kwargs["user_metadata"] = user_metadata
+        kwargs["system_config"] = sys_config
 
         request = Scans.prepare_scan_request(self.scan_name, self.scan_info, *args, **kwargs)
         request_id = str(uuid.uuid4())
@@ -265,8 +263,9 @@ class Scans:
                     )
 
         metadata = {}
-        if "metadata" in kwargs:
-            metadata = kwargs.pop("metadata")
+        metadata.update(kwargs["system_config"])
+        metadata["user_metadata"] = kwargs.pop("user_metadata", {})
+
         params = {"args": Scans._parameter_bundler(args, bundle_size), "kwargs": kwargs}
         # check the number of arg bundles against the number of required bundles
         if bundle_size:
@@ -408,17 +407,20 @@ class DatasetIdOnHold(ContextDecorator):
         queue.next_dataset_number += 1
 
 
-class FileWriterData:
+class FileWriter:
     @typechecked
-    def __init__(self, fw_config: Dict[Literal["file_suffix", "file_directory"], str]) -> None:
+    def __init__(self, file_suffix: str = None, file_directory: str = None) -> None:
         """Context manager for updating metadata
 
         Args:
             fw_config (dict): Dictionary with metadata for the filewriter, can only have keys "file_suffix" and "file_directory"
         """
         self.client = self._get_client()
-        self._fw_config = fw_config
+        self.system_config = self.client.system_config
+        self._orig_system_config = None
         self._orig_metadata = None
+        self.file_suffix = file_suffix
+        self.file_directory = file_directory
 
     def _get_client(self):
         """Get BEC client"""
@@ -427,12 +429,16 @@ class FileWriterData:
     def __enter__(self):
         """Enter the context manager"""
         self._orig_metadata = deepcopy(self.client.metadata)
-        self.client.metadata.update(self._fw_config)
+        self._orig_system_config = self.system_config.model_copy(deep=True)
+        self.system_config.file_suffix = self.file_suffix
+        self.system_config.file_directory = self.file_directory
         return self
 
-    def exit(self, *exc):
+    def __exit__(self, *exc):
         """Exit the context manager"""
         self.client.metadata = self._orig_metadata
+        self.system_config.file_suffix = self._orig_system_config.file_suffix
+        self.system_config.file_directory = self._orig_system_config.file_directory
 
 
 class Metadata:

bec_lib/tests/test_client.py

@@ -0,0 +1,22 @@
+""" This module tests the bec_lib.client module. """
+
+import pytest
+
+from bec_lib.client import SystemConfig
+
+
+def test_system_config():
+    """Test the SystemConfig class."""
+    config = SystemConfig(file_suffix="suff", file_directory="dir")
+    assert config.file_suffix == "suff"
+    assert config.file_directory == "dir"
+    config = SystemConfig()
+    assert config.file_suffix is None
+    assert config.file_directory is None
+    config.file_suffix = "suff_-"
+    config.file_directory = "/dir_-/blabla"
+    assert config.file_suffix == "suff_-"
+    assert config.file_directory == "dir_-/blabla"
+    with pytest.raises(ValueError):
+        config = SystemConfig(file_suffix="@")
+        config = SystemConfig(file_directory="ä")

bec_lib/tests/test_scan_context.py

@@ -3,7 +3,15 @@ from unittest import mock
 import pytest
 
 from bec_lib.device import DeviceBase
-from bec_lib.scans import DatasetIdOnHold, HideReport, Metadata, ScanDef, ScanExport, ScanGroup
+from bec_lib.scans import (
+    DatasetIdOnHold,
+    FileWriter,
+    HideReport,
+    Metadata,
+    ScanDef,
+    ScanExport,
+    ScanGroup,
+)
 
 # pylint: disable=no-member
 # pylint: disable=missing-function-docstring
@@ -11,6 +19,18 @@ from bec_lib.scans import DatasetIdOnHold, HideReport, Metadata, ScanDef, ScanEx
 # pylint: disable=protected-access
 
 
+def test_filewriter_cm(bec_client_mock):
+    client = bec_client_mock
+    client.scans._file_writer = None
+    client.system_config.file_directory = None
+    client.system_config.file_suffix = None
+    with FileWriter(file_suffix="testsuffix", file_directory="testdirectory"):
+        assert client.system_config.file_directory == "testdirectory"
+        assert client.system_config.file_suffix == "testsuffix"
+    assert client.system_config.file_directory is None
+    assert client.system_config.file_suffix is None
+
+
 def test_metadata_handler(bec_client_mock):
     client = bec_client_mock
     client.metadata = {"descr": "test", "uid": "12345"}

bec_lib/tests/test_scan_object.py

@@ -88,7 +88,7 @@ def test_scan_object_file_suffix(scan_obj, dev):
                 step=0.5,
                 exp_time=0.1,
                 relative=False,
-                metadata={"file_suffix": "testsample"},
+                file_suffix="testsample",
             )
             assert scan_report.call_args.args[0].metadata["file_suffix"] == "testsample"
 
@@ -124,7 +124,7 @@ def test_scan_object_raises_on_non_ascii_chars(scan_obj, dev, file_suffix, file_
                         step=0.5,
                         exp_time=0.1,
                         relative=False,
-                        metadata={"file_suffix": file_suffix},
+                        file_suffix=file_suffix,
                     )
             else:
                 scan_obj._run(
@@ -137,7 +137,7 @@ def test_scan_object_raises_on_non_ascii_chars(scan_obj, dev, file_suffix, file_
                     step=0.5,
                     exp_time=0.1,
                     relative=False,
-                    metadata={"file_suffix": file_suffix},
+                    file_suffix=file_suffix,
                 )
                 assert scan_report.call_args.args[0].metadata["file_suffix"] == file_suffix
 
@@ -168,7 +168,7 @@ def test_scan_object_raises_on_non_ascii_chars_dir(scan_obj, dev, file_dir, file
                         step=0.5,
                         exp_time=0.1,
                         relative=False,
-                        metadata={"file_directory": file_dir},
+                        file_directory=file_dir,
                     )
             else:
                 scan_obj._run(
@@ -181,7 +181,7 @@ def test_scan_object_raises_on_non_ascii_chars_dir(scan_obj, dev, file_dir, file
                     step=0.5,
                     exp_time=0.1,
                     relative=False,
-                    metadata={"file_directory": file_dir},
+                    file_directory=file_dir,
                 )
                 assert scan_report.call_args.args[0].metadata["file_directory"] == file_dir.strip(
                     "/"
@@ -201,7 +201,10 @@ def test_scan_object_receives_sample_name(scan_obj, dev):
             scan_obj.client, "get_global_var", side_effect=get_global_var_side_effect
         ):
             scan_obj._run(dev.samx, -5, 5, dev.samy, -5, 5, step=0.5, exp_time=0.1, relative=False)
-            assert scan_report.call_args.args[0].metadata["sample_name"] == "test_sample"
+            assert (
+                scan_report.call_args.args[0].metadata["user_metadata"]["sample_name"]
+                == "test_sample"
+            )
 
 
 def test_scan_object_receives_scan_group(scan_obj, dev):

docs/source/conf.py

@@ -59,6 +59,7 @@ myst_enable_extensions = [
 autosummary_generate = True  # Turn on sphinx.ext.autosummary
 add_module_names = False  # Remove namespaces from class/method signatures
 autodoc_inherit_docstrings = True  # If no docstring, inherit from base class
+autodoc_mock_imports = ["pydantic"]
 set_type_checking_flag = True  # Enable 'expensive' imports for sphinx_autodoc_typehints
 autoclass_content = "both"  # Include both class docstring and __init__
 

docs/source/developer/developer.md

@@ -20,7 +20,7 @@ glossary/
 ***
 
 ````{grid} 2
-:gutter: 6
+:gutter: 5
 
 ```{grid-item-card}
 :link: developer.getting_started

docs/source/developer/file_writer/file_writer.md

@@ -2,22 +2,31 @@
 ## File Writer
 BEC’s file writer is a dedicated service that writes HDF5 files with Nexus-compatible metadata entries to disk. It also adds external links to files written by other services, such as data backends for large 2D detector data. The internal structure of the files can be adjusted to the beamline’s needs using customizable plugins to comply with the desired [NeXus application definition](https://manual.nexusformat.org/classes/applications/index.html).
 
-When the service starts, a `base_path` is configured, and all data can only be written to disk relative to this path. By default, the relative path follows the template `/data/S00000-S00999/S00001/S00001_master.h5` for scan number 1. To compile the appropriate path for secondary services, we provide the utility class [`bec_lib.file_utils.FileWriter`](/api_reference/_autosummary/bec_lib.file_utils.FileWriter.rst#bec_lib.file_utils.FileWriter) with the method [`compile_full_filename`](/api_reference/_autosummary/bec_lib.file_utils.FileWriter.rst#bec_lib.file_utils.FileWriter.compile_full_filename), which automatically prepares the correct filepath. 
+When the service starts, a **base_path** is configured, and all data can only be written to disk relative to this path. By default, the relative path follows the template `/data/S00000-S00999/S00001/S00001_master.h5` for scan number 1. To compile the appropriate path for secondary services, we provide the utility class [`bec_lib.file_utils.FileWriter`](/api_reference/_autosummary/bec_lib.file_utils.FileWriter.rst#bec_lib.file_utils.FileWriter) with the method [`compile_full_filename`](/api_reference/_autosummary/bec_lib.file_utils.FileWriter.rst#bec_lib.file_utils.FileWriter.compile_full_filename), which automatically prepares the correct filepath. 
 If secondary services within *ophyd_devices* need to be configured with the appropriate file path, we recommend using this function since it will ensure that all custom changes to the file name and directory will be properly compiled and returned.
 
-### Changing the File Directory or Adding a Suffix
+### Changing the file directory or adding a suffix
 
-The relative filepath can be configured and adapted dynamically. We use the metadata provided within BEC to inform the file writer about these changes. For this purpose, we reserve the keys `file_suffix` and `file_directory` in the metadata for scans. We note that both variables must only contain *alphanumeric ASCII* characters.
-To configure these variables, we offer three different options:
+The relative filepath can be configured and adapted dynamically. 
+We provide the possibility to change the file directory or add a suffix to the file name through a **system_config**.
+Keep in mind that the file_directory will always be considered relative to the **base_path**. Providing an absolute path by accident will be transformed into a relative path. 
+In addition, both file_suffix and file_directory may only contain alphanumeric ASCII characters and the following special characters: '-', '_' and '/'. 
+This will be automatically checked, and raise for an invalid input.
+You may use one of the two options below:
 
-1. **Handing the adjusted metadata directly to the scan command**: This method will only be considered for the given scan command but has the highest priority.
-    ```python
-    scans.line_scan(dev.samx, -5, 5, steps=100, relativ=True, metadata={'file_suffix': 'sampleA', 'file_directory': 'study1337'})
-    ```
+1. **Changing the system_config**
 
-2. **Adding the information to `bec.metadata` in the command line interface**: This information will be considered for all following scans, unless explicitly overridden or deleted by a scan command.
-    ```python
-    bec.metadata.update({'file_suffix': 'sampleA', 'file_directory': 'study1337'})
-    ```
+The **system_config** is accessible via [`bec.system_config`](/api_reference/_autosummary/bec_lib.client.SystemConfig) and can be used to change file_suffix and file_directory. It is directly exposed to users via the client. Changing the *file_suffix* and *file_directory* will be considered for all following scans.
+```python
+bec.system_config.file_suffix = 'sampleA'
+bec.system_config.file_directory = 'my_dir/my_setup'
+```
+Assuming the *basepath* to be `'/bec/data'` and a `scannr = 101`, the file writer now writes to the following filepath: `'/bec/data/my_dir/my_setup/S00101_master_sampleA.h5'`.
+If you only provide *file_suffix*, but no additional *file_directory*, the filepath will be:`'/bec/data/S00000-S00999/S00101_sampleA/S00101_master_sampleA.h5'`. 
 
-3. **Using global variables [`bec.set_global_var`](/api_reference/_autosummary/bec_lib.client.BECClient.rst#bec_lib.client.BECClient.set_global_var)**: You can set `file_suffix` and `file_directory` as global variables within BEC, and they will be automatically considered. Note, this has the lowest priority compared to options 1 or 2.
+2.  **Adding additional arguments to a scan**
+
+You can also add the *file_suffix* and *file_directory* as arguments to the scan command. This will only affect the current scan and will not be considered for following scans. Adding the arguments to the scan command has priority and will override the information provided in *system_config*.
+```python
+scans.line_scan(dev.samx, -5, 5, steps=1, relative=True, file_suffix='sampleA', file_directory='my_dir/my_setup')
+```