Initial commit

Signed-off-by: Filip Leonarski <filip.leonarski@psi.ch>

python/__init__.py

@@ -0,0 +1 @@
+

python/jungfraujoch.py

@@ -0,0 +1,173 @@
+#  Copyright (2019-2022) Paul Scherrer Institute
+#  SPDX-License-Identifier: GPL-3.0-or-later
+
+import requests
+
+
+class DetectorClient:
+    def __init__(self, hostname: str, port: int):
+        self.__addr = "http://{}:{}/".format(hostname, port)
+
+    def __post_request(self, prefix: str, content: dict) -> None:
+        api_url = self.__addr + "/" + prefix
+        response = requests.post(api_url, json=content)
+        response.raise_for_status()
+
+    def __put_request(self, prefix: str, content: dict) -> None:
+        api_url = self.__addr + "/" + prefix
+        response = requests.put(api_url, json=content)
+        response.raise_for_status()
+
+    def __get_request(self, prefix: str) -> dict:
+        api_url = self.__addr + "/" + prefix
+        response = requests.get(api_url)
+        response.raise_for_status()
+        return response.json()
+
+    def initialize(self) -> None:
+        """
+        Need to be performed before starting measurement.
+        It is also used to recover detector from error condition.
+
+        Initialize will execute calibration procedure.
+        """
+        self.__post_request("/detector/initialize", dict())
+
+    def deactivate(self) -> None:
+        """
+        Prepare the detector to be turned off (turn off high voltage and ASIC).
+        """
+        self.__post_request("/detector/deactivate", dict())
+
+    def pedestal(self) -> None:
+        """
+        Calibration procedure
+        """
+        self.__post_request("/detector/pedestal", dict())
+
+    def start(self, dataset_settings: dict) -> None:
+        """
+        Start measurement
+
+        :param dataset_settings: Dictionary containing dataset specific parameters:
+        ["images_per_trigger"] - number of images collected per trigger;
+        ["ntrigger"] - number of triggers detector should expect;
+        ["summation"] or ["image_time_us"] - relation between internal frame time and image time
+        (use only one of the two options; if image time is used, it has to be multiple of frame time);
+        ["beam_x_pxl"] - beam center location on X-axis in pixels;
+        ["beam_y_pxl"] - beam center location on Y-axis in pixels;
+        ["detector_distance_mm"] - detector distance from the sample in mm;
+        ["photon_energy_keV"] - photon energy;
+        ["file_prefix"] - prefix for HDF5 files (empty == no files are written, e.g. for preview);
+        ["data_file_count"] - split data frames into N-files on a round-robin basis;
+        ["compression"] - algorithm for compression (NO_COMPRESSION, BSHUF_LZ4, BSHUF_ZSTD, and BSHUF_ZSTD_RLE);
+        ["sample_name"] - name of sample, can work as merge ID;
+        ["unit_cell"] - unit cell (dictionary with a, b, c, alpha, beta, gamma);
+        ["scattering vector"] - optional (default is [0, 0, 1], array of 3 floats
+        """
+        self.__post_request("/detector/start", dataset_settings)
+
+    def trigger(self) -> None:
+        """
+        Send soft trigger to the detector
+        """
+        self.__post_request("/detector/trigger", dict())
+
+    def stop(self) -> None:
+        """
+        Block till measurement is finished (it is not necessary to call stop to start next measurement).
+        """
+        self.__post_request("/detector/stop", dict())
+
+    def cancel(self) -> None:
+        """
+        Cancel running data collection.
+        """
+        self.__post_request("/detector/cancel", dict())
+
+    @property
+    def detector_settings(self) -> dict:
+        """
+        Get settings of the detector that require to redo the calibration
+
+        :return Dictionary with detector settings (see setter for more information)
+        """
+        return self.__get_request("/detector/settings")
+
+    @detector_settings.setter
+    def detector_settings(self, settings: dict):
+        """
+        Put settings of the detector that require to redo the calibration.
+        Can only be done when detector is idle and change will trigger pedestal calibration.
+
+        :param settings: Dictionary with the following entries:
+        ["frame_time_us"] - internal frame time (recommended is using only 1000 and 500 us values);
+        ["count_time_us"] - count time (must be less by at least 20 us than frame time, zero means the longest possible
+        for the given frame time);
+        ["use_storage_cells"] - when True enables usage of all 16 storage cells;
+        ["use_internal_packet_generator"] - take frames from the internal packet generator of the FPGA
+        (to test FPGA if detector is not present);
+        ["collect_raw_data"] - when True JUNGFAU images won't be converted to photon counts;
+        ["pedestal_g0_frames"] - number of frames used to calculate G0 pedestal
+        (optional, if not provided current value won't be changed);
+        ["pedestal_g1_frames"] - number of frames used to calculate G1 pedestal
+        (optional, if not provided current value won't be changed);
+        ["pedestal_g2_frames"] - number of frames used to calculate G2 pedestal
+        (optional, if not provided current value won't be changed);
+        """
+        self.__put_request("/detector/settings", settings)
+
+    @property
+    def data_processing_settings(self) -> dict:
+        """
+        Get settings of the data processing
+
+        :return Dictionary with data processing settings (see setter for more information)
+        """
+
+        return self.__get_request("/data_processing/settings")
+
+    @data_processing_settings.setter
+    def data_processing_settings(self, settings: dict):
+        """
+        Configure real-time data processing settings, safe to do anytime, including during measurement
+
+        :param settings: Dictionary with the following entries:
+
+        """
+
+        self.__put_request("/data_processing/settings", settings)
+
+    @property
+    def last_measurement_stats(self):
+        """
+        Get statistics for the last measurement
+
+        :return Dictionary with the following entries:
+        ["file_prefix"] - file prefix for data collection;
+        ["images_collected"] - number of images successfully collected;
+        ["max_image_number_sent"] - the highest image number
+        (can differ from image number, if images were lost during collection);
+        ["collection_efficiency"] - fraction of expected packets collected (1.0 is all good);
+        ["compression_ratio"] - compression ratio of the dataset;
+        ["cancelled"] - if cancellation happened during data collection;
+        ["max_receive_delay"] - number of frames receiver was late vs. detector - large number means there were problems;
+        ["writer_performance_MBs"] - total writing speed (only present if files were written);
+        ["images_written"] - number of images written (should be the same as images_collected,
+        only present if files were written);
+        ["indexing_rate"] - indexing rate from real-time indexing;
+        ["indexing_performance_ms"] - average time to index a single image;
+        """
+        return self.__get_request("/detector/last_measurement")
+
+    @property
+    def status(self):
+        """
+        Get detector status:
+
+        :return Dictionary with the following entries:
+        ["broker_state"] - current state: NOT_INITIALIZED, IDLE, BUSY, PEDESTAL, DATA_COLLECTION, ERROR;
+        ["progress'] - data collection progress in percent (only if measurement running);
+        ["indexing_rate"] - instantaneous indexing rate from real-time indexing (only if measurement running);
+        """
+        return self.__get_request("/detector/status")

python/jungfraujoch_metadata.py

@@ -0,0 +1,121 @@
+#  Copyright (2019-2022) Paul Scherrer Institute
+#  SPDX-License-Identifier: GPL-3.0-or-later
+
+import h5py
+import sys
+import gemmi
+
+
+def wvl_a_to_ev(val: float) -> float:
+    """
+    Convert wavelength in A to photon energy in eV
+    """
+    return 12398.4 / val
+
+
+def extract_metadata_nxmx(nxmx_file: str) -> dict:
+    """
+    Create a dictionary of metadata based on HDF5 master file
+    """
+    output = {}
+    with h5py.File(nxmx_file, "r") as f:
+        output["filename"] = nxmx_file
+        output["photon_energy_eV"] = wvl_a_to_ev(
+            f["/entry/instrument/beam/incident_wavelength"][()]
+        )
+        output["pixel_size_m"] = f["/entry/instrument/detector/x_pixel_size"][()]
+
+        if "/entry/instrument/detector/underload_value" in f:
+            output["underload"] = f["/entry/instrument/detector/underload_value"][()]
+        output["saturation"] = f["/entry/instrument/detector/saturation_value"][()]
+
+        output["detector_distance_m"] = f[
+            "/entry/instrument/detector/detector_distance"
+        ][()]
+        output["beam_center_x"] = f["/entry/instrument/detector/beam_center_x"][()]
+        output["beam_center_y"] = f["/entry/instrument/detector/beam_center_y"][()]
+
+        output["width"] = f["/entry/data/data"].shape[2]
+        output["height"] = f["/entry/data/data"].shape[1]
+
+        output["image_time_s"] = f["/entry/instrument/detector/frame_time"][()]
+
+        output["sample_name"] = f["/entry/sample/name"][()]
+        if "/entry/sample/unit_cell" in f:
+            output["unit_cell"] = f["/entry/sample/unit_cell"][:6]
+
+        if "/entry/sample/space_group" in f:
+            output["space_group"] = f["/entry/sample/space_group"]
+    return output
+
+
+def nxmx_to_geom(metadata_nxmx: dict, output_file: str) -> None:
+    """
+    Write dictionary generated by extract_metadata_nxmx to CrystFEL geometry file
+    """
+    output = {}
+
+    output["photon_energy"] = "{:f}".format(metadata_nxmx["photon_energy_eV"])
+    output["adu_per_eV"] = "{:.10f}".format(1 / metadata_nxmx["photon_energy_eV"])
+    output["clen"] = "{:f}".format(metadata_nxmx["detector_distance_m"])
+    output["res"] = "{:f}".format(1.0 / metadata_nxmx["pixel_size_m"])
+
+    if "underload" in metadata_nxmx:
+        output["flag_lessthen"] = metadata_nxmx["underload"] + 1
+
+    output["rigid_group_0"] = 0
+    output["rigid_group_collection_0"] = 0
+    output["data"] = "/entry/data/data"
+    output["dim0"] = "%"
+    output["dim1"] = "ss"
+    output["dim2"] = "fs"
+
+    output["mask_file"] = metadata_nxmx["filename"]
+
+    output["mask"] = "/entry/instrument/detector/pixel_mask"
+    output["mask_good"] = "0x0"
+    output["mask_bad"] = "0xffffffff"
+
+    output["0/min_fs"] = 0
+    output["0/min_ss"] = 0
+    output["0/max_fs"] = metadata_nxmx["width"]
+    output["0/max_ss"] = metadata_nxmx["height"]
+
+    output["0/corner_x"] = "{:f}".format(-1.0 * metadata_nxmx["beam_center_x"])
+    output["0/corner_y"] = "{:f}".format(-1.0 * metadata_nxmx["beam_center_y"])
+    output["0/fs"] = "x"
+    output["0/ss"] = "y"
+
+    with open(output_file, "w") as f:
+        for i in output:
+            f.write("{} = {}\n".format(i, output[i]))
+
+
+def nxmx_to_cell(metadata_nxmx: dict, output_file: str):
+    """
+    Write dictionary generated by extract_metadata_nxmx to CrystFEL cell file (PDB)
+    """
+    unit_cell_str = "{:9.3f}{:9.3f}{:9.3f}{:7.2f}{:7.2f}{:7.2f}".format(
+        *metadata_nxmx["unit_cell"]
+    )
+    if "space_group" in metadata_nxmx:
+        space_group_str = gemmi.find_spacegroup_by_number(
+            metadata_nxmx["space_group"]
+        ).hm
+    else:
+        space_group_str = gemmi.find_spacegroup_by_number(1).hm
+    with open(output_file, "w") as f:
+        f.write("CRYST1{:s} {:11s}{:4d}\n".format(unit_cell_str, space_group_str, 1))
+
+
+if __name__ == "__main__":
+    if (len(sys.argv) != 4) and (len(sys.argv) != 3):
+        print(
+            "Usage ./jungfraujoch_metadata.py <JUNGFRAU/EIGER master file> <CrystFEL output file> {<cell file>}"
+        )
+        sys.exit(1)
+    else:
+        metadata = extract_metadata_nxmx(sys.argv[1])
+        nxmx_to_geom(metadata, sys.argv[2])
+        if len(sys.argv) == 4:
+            nxmx_to_cell(metadata, sys.argv[3])

python/preview.py

@@ -0,0 +1,68 @@
+import json
+import zmq
+import numpy
+import sys
+import signal
+import base64
+
+sys.path.insert(0, "/opt/dectris/albula/4.1/bin")
+sys.path.insert(0, "/opt/dectris/albula/4.1/python")
+
+import dectris.albula as albula
+from dectris.albula import DNoObject, DDrawingEllipse, DDrawingString, DDrawingLine
+
+
+TERMINATE = False
+
+
+def signal_handler(signal, frame):
+    global TERMINATE
+    TERMINATE = True
+
+
+signal.signal(signal.SIGINT, signal_handler)
+
+context = zmq.Context()
+socket = context.socket(zmq.SUB)
+socket.RCVTIMEO = 1000 # in milliseconds
+socket.connect("tcp://xbl-daq-38:5400")
+socket.setsockopt(zmq.SUBSCRIBE, b"")
+
+OPTIONAL_DATA = albula.DImageOptionalData()
+
+OPTIONAL_DATA.set_x_pixel_size(0.000075)
+OPTIONAL_DATA.set_y_pixel_size(0.000075)
+
+albulaMain = albula.openMainFrame()
+albulaSubFrame = albulaMain.openSubFrame()
+
+
+def read_message():
+    try:
+        msg = socket.recv_json()
+        OPTIONAL_DATA.set_wavelength(msg["wavelength_A"])
+        OPTIONAL_DATA.set_beam_center_x(msg["beam_x_pxl"])
+        OPTIONAL_DATA.set_beam_center_y(msg["beam_y_pxl"])
+        OPTIONAL_DATA.set_detector_distance(msg["detector_distance_mm"] * 0.001)
+        OPTIONAL_DATA.set_saturation_value(int(msg["saturation_value"]))
+        if int(msg["pixel_depth"]) == 2:
+            image_array = numpy.frombuffer(base64.b64decode(msg["data"]), numpy.int16)
+        else:
+            image_array = numpy.frombuffer(base64.b64decode(msg["data"]), numpy.int32)
+        height = int(msg["height"])
+        width = int(msg["width"])
+        image_number = int(msg["image_number"])
+        image_array = numpy.reshape(image_array, (height, width))
+        print("Received image %d %d %d %d"%(image_number, height, width, len(image_array)))
+        dimage = albula.DImage(image_array)
+        if image_number >= 0:
+            dimage.setOptionalData(OPTIONAL_DATA)
+            albulaSubFrame.loadImage(dimage)
+            albulaSubFrame.setTitle("JUNGFRAU PREVIEW Dataset: %s Image: %d" % (msg["file_prefix"], image_number))
+        else:
+            OPTIONAL_DATA.set_pixel_mask(dimage)
+    except:
+        pass
+
+while not TERMINATE:
+    read_message()