fix(pandabox): fix conversion of data to ophyd signal dict.

device_test_reports/report_demo_config.txt

@@ -0,0 +1,434 @@
+Checking eiger...
+OK
+Checking dyn_signals...
+OK
+Checking pseudo_signal1...
+OK
+Checking hexapod...
+OK
+Checking eyefoc...
+OK
+Checking eyex...
+OK
+Checking eyey...
+OK
+Checking flyer_sim...
+OK
+Checking hrox...
+OK
+Checking hroy...
+OK
+Checking hroz...
+OK
+Checking hx...
+OK
+Checking hy...
+OK
+Checking hz...
+OK
+Checking mbsx...
+OK
+Checking mbsy...
+OK
+Checking pinx...
+OK
+Checking piny...
+OK
+Checking pinz...
+OK
+Checking samx...
+OK
+Checking samy...
+OK
+Checking samz...
+OK
+Checking bpm3a...
+OK
+Checking bpm3b...
+OK
+Checking bpm3c...
+OK
+Checking bpm3d...
+OK
+Checking bpm3i...
+OK
+Checking bpm3x...
+OK
+Checking bpm3y...
+OK
+Checking bpm3z...
+OK
+Checking bpm4a...
+OK
+Checking bpm4b...
+OK
+Checking bpm4c...
+OK
+Checking bpm4d...
+OK
+Checking bpm4i...
+OK
+Checking bpm4s...
+OK
+Checking bpm4x...
+OK
+Checking bpm4xf...
+OK
+Checking bpm4xm...
+OK
+Checking bpm4y...
+OK
+Checking bpm4yf...
+OK
+Checking bpm4ym...
+OK
+Checking bpm4z...
+OK
+Checking bpm5a...
+OK
+Checking bpm5b...
+OK
+Checking bpm5c...
+OK
+Checking bpm5d...
+OK
+Checking bpm5i...
+OK
+Checking bpm5x...
+OK
+Checking bpm5y...
+OK
+Checking bpm5z...
+OK
+Checking bpm6a...
+OK
+Checking bpm6b...
+OK
+Checking bpm6c...
+OK
+Checking bpm6d...
+OK
+Checking bpm6i...
+OK
+Checking bpm6x...
+OK
+Checking bpm6y...
+OK
+Checking bpm6z...
+OK
+Checking curr...
+OK
+Checking diode...
+OK
+Checking ebpmdx...
+OK
+Checking ebpmdy...
+OK
+Checking ebpmux...
+OK
+Checking ebpmuy...
+OK
+Checking ftp...
+OK
+Checking temp...
+OK
+Checking transd...
+OK
+Checking aptrx...
+OK
+Checking aptry...
+OK
+Checking bim2x...
+OK
+Checking bim2y...
+OK
+Checking bm1trx...
+OK
+Checking bm1try...
+OK
+Checking bm2trx...
+OK
+Checking bm2try...
+OK
+Checking bm3trx...
+OK
+Checking bm3try...
+OK
+Checking bm4trx...
+OK
+Checking bm4try...
+OK
+Checking bm5trx...
+OK
+Checking bm5try...
+OK
+Checking bm6trx...
+OK
+Checking bm6try...
+OK
+Checking bpm4r...
+OK
+Checking bpm5r...
+OK
+Checking bs1x...
+OK
+Checking bs1y...
+OK
+Checking bs2x...
+OK
+Checking bs2y...
+OK
+Checking burstn...
+OK
+Checking burstr...
+OK
+Checking ddg1a...
+OK
+Checking ddg1b...
+OK
+Checking ddg1c...
+OK
+Checking ddg1d...
+OK
+Checking ddg1e...
+OK
+Checking ddg1f...
+OK
+Checking ddg1g...
+OK
+Checking ddg1h...
+OK
+Checking dettrx...
+OK
+Checking di2trx...
+OK
+Checking di2try...
+OK
+Checking dtpush...
+OK
+Checking dtth...
+OK
+Checking dttrx...
+OK
+Checking dttry...
+OK
+Checking dttrz...
+OK
+Checking ebcsx...
+OK
+Checking ebcsy...
+OK
+Checking ebfi1...
+OK
+Checking ebfi2...
+OK
+Checking ebfi3...
+OK
+Checking ebfi4...
+OK
+Checking ebfzpx...
+OK
+Checking ebfzpy...
+OK
+Checking ebtrx...
+OK
+Checking ebtry...
+OK
+Checking ebtrz...
+OK
+Checking fi1try...
+OK
+Checking fi2try...
+OK
+Checking fi3try...
+OK
+Checking fsh1x...
+OK
+Checking fsh2x...
+OK
+Checking ftrans...
+OK
+Checking fttrx1...
+OK
+Checking fttrx2...
+OK
+Checking fttry1...
+OK
+Checking fttry2...
+OK
+Checking fttrz...
+OK
+Checking idgap...
+OK
+Checking mibd...
+OK
+Checking mibd1...
+OK
+Checking mibd2...
+OK
+Checking miroll...
+OK
+Checking mith...
+OK
+Checking mitrx...
+OK
+Checking mitry...
+OK
+Checking mitry1...
+OK
+Checking mitry2...
+OK
+Checking mitry3...
+OK
+Checking mobd...
+OK
+Checking mobdai...
+OK
+Checking mobdbo...
+OK
+Checking mobdco...
+OK
+Checking mobddi...
+OK
+Checking mokev...
+OK
+Checking mopush1...
+OK
+Checking mopush2...
+OK
+Checking moroll1...
+OK
+Checking moroll2...
+OK
+Checking moth1...
+OK
+Checking moth1e...
+OK
+Checking moth2...
+OK
+Checking moth2e...
+OK
+Checking motrx2...
+OK
+Checking motry...
+OK
+Checking motry2...
+OK
+Checking motrz1...
+OK
+Checking motrz1e...
+OK
+Checking moyaw2...
+OK
+Checking sl0ch...
+OK
+Checking sl0trxi...
+OK
+Checking sl0trxo...
+OK
+Checking sl0wh...
+OK
+Checking sl1ch...
+OK
+Checking sl1cv...
+OK
+Checking sl1trxi...
+OK
+Checking sl1trxo...
+OK
+Checking sl1tryb...
+OK
+Checking sl1tryt...
+OK
+Checking sl1wh...
+OK
+Checking sl1wv...
+OK
+Checking sl2ch...
+OK
+Checking sl2cv...
+OK
+Checking sl2trxi...
+OK
+Checking sl2trxo...
+OK
+Checking sl2tryb...
+OK
+Checking sl2tryt...
+OK
+Checking sl2wh...
+OK
+Checking sl2wv...
+OK
+Checking sl3ch...
+OK
+Checking sl3cv...
+OK
+Checking sl3trxi...
+OK
+Checking sl3trxo...
+OK
+Checking sl3tryb...
+OK
+Checking sl3tryt...
+OK
+Checking sl3wh...
+OK
+Checking sl3wv...
+OK
+Checking sl4ch...
+OK
+Checking sl4cv...
+OK
+Checking sl4trxi...
+OK
+Checking sl4trxo...
+OK
+Checking sl4tryb...
+OK
+Checking sl4tryt...
+OK
+Checking sl4wh...
+OK
+Checking sl4wv...
+OK
+Checking sl5ch...
+OK
+Checking sl5cv...
+OK
+Checking sl5trxi...
+OK
+Checking sl5trxo...
+OK
+Checking sl5tryb...
+OK
+Checking sl5tryt...
+OK
+Checking sl5wh...
+OK
+Checking sl5wv...
+OK
+Checking strox...
+OK
+Checking stroy...
+OK
+Checking stroz...
+OK
+Checking sttrx...
+OK
+Checking sttry...
+OK
+Checking ring_current_sim...
+OK
+Checking monitor_async...
+OK
+Checking rt_controller...
+OK
+Checking waveform...
+OK
+
+
+
+========================================
+Summary:
+All devices passed the test.

ophyd_devices/devices/panda_box/README.md

@@ -0,0 +1,55 @@
+# PandaBox Integration
+Short Doumentation of the PandaBox Device Integration in Ophyd Devices
+
+## Overview
+
+The PandaBox integration provides a base class for interfacing with PandaBox hardware from Diamond Light Source. This implementation wraps the PandaBox hardware as a `PSIDeviceBase` device, integrating it into the BEC scan interface. It uses the `pandablocks` library for communication and data acquisition. Beamline-specific implementations should use the *on_hook* methods from PSIDeviceBase to implement custom logic.
+
+**IMPORTANT** : If the `on_connected()` method is overridden by a child class, it must always call `super().on_connected()` first to ensure proper initialization of the PandaBox device. This is implemented in the [PandaBox class](./panda_box.py). 
+
+### PandaState
+
+The PandaBox has a PCAP module that can be used to record block values. The base integration implements logic that automatically arms/disarms the PCAP module for the BEC scan interface. Callbacks can be attached for status handlings *add_status_callback* and data handling *add_data_callback* respectively. Below is the enum defining the various PandaBox states:
+
+```python
+class PandaState(StrEnum):
+    READY = "ready"      # Ready for data acquisition
+    START = "start"      # Data acquisition started
+    FRAME = "frame"      # Frame data received
+    END = "end"          # Data acquisition ended
+    DISARMED = "disarmed"  # Device is disarmed
+```
+
+### Public API (USER ACCESS)
+
+There are a couple of methods which are tagged as USER_ACCESS methods, and thereby also available on the proxy devices. 
+These methods include:
+ - `send_raw(cmd: Union[str, list[str]]) -> Any` : Send raw commands or lists of commands to the PandaBox hardware.
+ - `add_status_callback(status: StatusBase, success: list[PandaState], failure: list[PandaState], check_directly: bool = True) -> str` : Register a callback to resolve status objects based on PandaBox events. PandaBox events are defined in the `PandaState` enum, which includes states like READY, START, FRAME, END, and DISARMED. These states correspond to different stages of that data acquisition of the PCAP module of the PandaBox. 
+- `remove_status_callback(cb_id: str) -> None` : Remove a registered status callback using its unique callback ID (str) which is returned by *add_status_callback*.
+- `add_data_callback(callback: Callable[[LITERAL_PANDA_DATA], None], data_type: PandaState = PandaState.FRAME.value) -> str` : Register a callback for processing PandaBox data. The callback function is called when data of the specified type (READY, START, FRAME, END, DATA) is received from the PandaBox. The default data type is FRAME, which corresponds to actual frame data from the PCAP module. These data frames can be inspected in pandablocks.response module.
+- `remove_data_callback(cb_id: str) -> None` : Remove a registered data callback using its unique callback ID (str) which is returned by *add_data_callback*.
+- `get_panda_state() -> str` : Get the current PandaBox data acquisition state as a string (ready, start, frame, end, data ).
+
+### Other useful methods
+
+- `_compile_frame_data_to_dict(frame_data: FrameData, signal_name_key_mapping: dict[str, str] | None = None) -> dict[str, Any]` : Convert FrameData from PandaBox into a dictionary format compatible with Ophyd signals. Optionally map PandaBox signal names to custom signal names.
+- `_get_signal_names_allowed_for_capture() -> list[str]` : Get a list of all signal keys that can be configured for capture on the PandaBox.
+- `_get_signal_names_configured_for_capture() -> list[str]` : Get a list of all signal keys that are currently configured for capture on the PandaBox.
+
+
+### Utility Scripts
+
+The module [utility_scripts.py](./utility_scripts.py) provides command-line tools for saving and loading PandaBox layouts to/from files. This is useful to save layouts configured via the PandaBox web interface, and store them alongside beamline-specific integrations of the PandaBox in the beamline plugin repository. Multiple layouts can be created and also loaded dynamically depending on the scan type. We recommend using this operation mode for beamline-specific use cases of the PandaBox.
+
+#### Save Layout from PandaBox to File
+``` bash
+python ./utility_scripts.py --host panda-box-host.psi.ch --save-layout ./my_layout.ini
+```
+Saves the current layout from the PandaBox at the specified host to a local file named `my_layout.ini`.
+
+#### Load Layout from File to PandaBox
+``` bash
+python ./utility_scripts.py --host panda-box-host.psi.ch --load-layout ./my_layout.ini
+```
+**IMPORTANT**: Loads the layout from the local file `my_layout.ini` to the PandaBox at the specified host. Please note that loading a layout will overwrite the current configuration on the PandaBox. The UI will partly update, but the WEB server needs to be restarted manually to reflect these changes properly. We expect beamlines to prepare and test layouts beforehands and not use the PandaBox web interface in operation. All dynamic configuration should be done through the ophyd device hooks either directly in the device integration or temporarily through custom scan implementations.

ophyd_devices/devices/panda_box/panda_box.py

@@ -21,10 +21,11 @@ the correct layout is loaded. Utility methods to load/save layouts to/from files
 from/to the PandaBox hardware are provided in this class too.
 """
 
+from __future__ import annotations
+
 import os
 import threading
 import uuid
-from collections import defaultdict
 from enum import StrEnum
 from typing import TYPE_CHECKING, Any, Callable, TypeAlias, Union
 
@@ -143,7 +144,7 @@ LITERAL_PANDA_COMMANDS: TypeAlias = Union[
     pbc.GetFieldInfo,
     pbc.GetPcapBitsLabels,
 ]
-LITERAL_PANDA_DATA: TypeAlias = Union[ReadyData, StartData, FrameData, EndData]
+LITERAL_PANDA_DATA: TypeAlias = Union[ReadyData, StartData, FrameData, EndData, Data]
 
 
 class PandaBox(PSIDeviceBase):
@@ -161,9 +162,9 @@ class PandaBox(PSIDeviceBase):
         *,
         name: str,
         host: str,
-        scan_info: "ScanInfo" | None = None,
+        scan_info: ScanInfo | None = None,
-        device_manager: "DeviceManagerDS" | None = None,
+        device_manager: DeviceManagerDS | None = None,
-        kwargs,
+        **kwargs,
     ) -> None:
         super().__init__(name=name, scan_info=scan_info, device_manager=device_manager, **kwargs)
         self.host = host
@@ -180,7 +181,7 @@ class PandaBox(PSIDeviceBase):
 
         # Thread to receive data from the PandaBox
         self.data_thread: threading.Thread = threading.Thread(
-            target=self._data_thread_loop, daemon=True
+            target=self._data_thread_loop, daemon=True, name=f"{self.name}_data_thread"
         )
         self.data_thread_kill_event = threading.Event()
         self.data_thread_run_event = threading.Event()
@@ -364,8 +365,8 @@ class PandaBox(PSIDeviceBase):
         # to ensure that we send the DISARM() command to the PandaBox to stop the acquisition cleanly. Multiple disarm
         # commands are safe to send, so we can always ensure that we disarm at the end of the readout loop. (TODO to check).
         """
-        with BlockingClient(self.host) as client:
+        try:
-            try:
+            with BlockingClient(self.host) as client:
                 for data in client.data(scaled=False):
                     if isinstance(data, ReadyData):
                         self._run_status_callbacks(PandaState.READY)
@@ -384,27 +385,27 @@ class PandaBox(PSIDeviceBase):
                         self._run_data_callbacks(data, PandaState.END.value)
                         break  # Exit data readout loop
 
-            finally:
+        finally:
-                # NOTE: This block ensures that we properly cleanup after a data acquisition,
+            # NOTE: This block ensures that we properly cleanup after a data acquisition,
-                # whether it completed successfully or was interrupted. This includes sending
+            # whether it completed successfully or was interrupted. This includes sending
-                # the DISARM() command to the PandaBox to stop any ongoing acquisition in case
+            # the DISARM() command to the PandaBox to stop any ongoing acquisition in case
-                # we exited the loop prematurely. It also clears the data_thread_run_event to block
+            # we exited the loop prematurely. It also clears the data_thread_run_event to block
-                # the data readout loop again, and runs the DISARMED status callbacks to notify
+            # the data readout loop again, and runs the DISARMED status callbacks to notify
-                # any registered status objects that the PandaBox is now disarmed. DISARMED is the
+            # any registered status objects that the PandaBox is now disarmed. DISARMED is the
-                # expected safe state of the data receiving loop from the PandaBox and was added
+            # expected safe state of the data receiving loop from the PandaBox and was added
-                # in addition to the existing READY, START, FRAME, END events created from the existing
+            # in addition to the existing READY, START, FRAME, END events created from the existing
-                # PandaBox data messages.
+            # PandaBox data messages.
 
-                client.send(self._disarm())  # Ensure we disarm at the end
+            self._disarm()  # Ensure we disarm at the end
 
-                self.data_thread_run_event.clear()  # Stop data readout loop
+            self.data_thread_run_event.clear()  # Stop data readout loop
 
-                self._run_status_callbacks(PandaState.DISARMED)  # Run DISARMED status callbacks
+            self._run_status_callbacks(PandaState.DISARMED)  # Run DISARMED status callbacks
 
-                # As DISARMED is not triggered by a data message, we manually run data callbacks for it here
+            # As DISARMED is not triggered by a data message, we manually run data callbacks for it here
-                # and run it with an empty Data() object following the base class for data message responses
+            # and run it with an empty Data() object following the base class for data message responses
-                # of the pandablocks library.
+            # of the pandablocks library.
-                self._run_data_callbacks(Data(), PandaState.DISARMED.value)
+            self._run_data_callbacks(Data(), PandaState.DISARMED.value)
 
     def _run_status_callbacks(self, event: PandaState) -> None:
         """
@@ -544,17 +545,20 @@ class PandaBox(PSIDeviceBase):
         """
         status = super().pre_scan()
         status_ready_data_received = StatusBase(obj=self)
+        self.cancel_on_stop(
+            status_ready_data_received
+        )  # Make sure we cancel if the scan is stopped
+        status_ready_data_received.add_callback(self._pre_scan_status_callback)
         self.add_status_callback(
             status=status_ready_data_received,
             success=[PandaState.READY],
             failure=[PandaState.FRAME, PandaState.END],
         )
-        status_ready_data_received.add_callback(self._pre_scan_status_callback)
+
         if status:
             ret_status = status_ready_data_received & status
         else:
             ret_status = status_ready_data_received
-        self.cancel_on_stop(ret_status)
         return ret_status
 
     def unstage(self) -> list[object] | StatusBase:
@@ -571,16 +575,18 @@ class PandaBox(PSIDeviceBase):
 
     def _get_signal_names_allowed_for_capture(self) -> list[str]:
         """Utility method to get a list of all signal keys that CAN BE CONFIGURED for capture on the PandaBox."""
-        ret = self.send_command(self.send_raw("*CAPTURE.*?"))
+        ret = self.send_raw("*CAPTURE.*?")
         # TODO check proper unpacking of returned keys
         return [key.split(" ")[0].strip("!") for key in ret if key.strip(".")]
 
     def _get_signal_names_configured_for_capture(self) -> list[str]:
         """Utility method to get a list of all signal keys thar ARE CURRENTLY CONFIGURED for capture on the PandaBox."""
-        ret = self.send_command(self.send_raw("*CAPTURE?"))
+        ret = self.send_raw("*CAPTURE?")
         return [key.split(" ")[0].strip("!") for key in ret if key.strip(".")]
 
-    def _compile_frame_data_to_dict(self, frame_data: FrameData) -> dict[str, Any]:
+    def _compile_frame_data_to_dict(
+        self, frame_data: FrameData, signal_name_key_mapping: dict[str, str] | None = None
+    ) -> dict[str, Any]:
         """
         Compile the data from a FrameData object into a dictionary with expected OPHYD
         read format, e.g. signal {signal_name: {"value": [...]}}.
@@ -591,12 +597,20 @@ class PandaBox(PSIDeviceBase):
         Returns:
             dict[str, Any]: The compiled data in OPHYD read format.
         """
-        out = defaultdict(list)
+        if signal_name_key_mapping is None:
+            signal_name_key_mapping = {}
+        # Create output dict
+        out = {}
         data = frame_data.data
         keys = data.dtype.names
+        # Map keys if mapping is provided
+        mapped_key = [signal_name_key_mapping.get(key, key) for key in keys]
+        # Initialize lists for each key, consider adjusting names to match
+        for k in mapped_key:
+            out[k] = {"value": []}  # Timestamp?
         for entry in data:
-            for i, key in enumerate(keys):
+            for i, k in enumerate(mapped_key):
-                out[key]["value"].append(entry[i])
+                out[k]["value"].append(entry[i])  # Fill values from data
 
     def _pre_scan_status_callback(self, status: StatusBase):
         """
@@ -605,9 +619,8 @@ class PandaBox(PSIDeviceBase):
         Args:
             status (StatusBase): The status object to resolve when arming is complete.
         """
-        if not status.done:
+        if status.done and status.success:
             self._arm()
-            status.set_finished()
 
     def _send_command(self, command: LITERAL_PANDA_COMMANDS) -> Any:
         """Send a command to the PandaBox via the BlockingClient."""

ophyd_devices/devices/panda_box/utility_scripts.py

@@ -11,7 +11,7 @@ Module with utility scripts to run on the PandaBox device.
 import argparse
 from pathlib import Path
 
-from ophyd_devices.devices.pandabox.pandabox import (
+from ophyd_devices.devices.panda_box.panda_box import (
     load_layout_from_file_to_panda,
     save_panda_layout_to_file,
 )