commit fad395dc6041957ee64db53bee4bdbc02c1755cb Author: Filip Leonarski (Gitea) Date: Thu Aug 13 18:43:50 2026 +0000 Deploy site diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 00000000..610dc968 --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 6836f6e9dbae1d0d2cfec53c6feec667 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/ACKNOWLEDGEMENT.doctree b/.doctrees/ACKNOWLEDGEMENT.doctree new file mode 100644 index 00000000..f57dc09e Binary files /dev/null and b/.doctrees/ACKNOWLEDGEMENT.doctree differ diff --git a/.doctrees/CBOR.doctree b/.doctrees/CBOR.doctree new file mode 100644 index 00000000..80883159 Binary files /dev/null and b/.doctrees/CBOR.doctree differ diff --git a/.doctrees/CHANGELOG.doctree b/.doctrees/CHANGELOG.doctree new file mode 100644 index 00000000..fdd02f65 Binary files /dev/null and b/.doctrees/CHANGELOG.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS.doctree b/.doctrees/CPU_DATA_ANALYSIS.doctree new file mode 100644 index 00000000..7bec6baf Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/DEPLOYMENT.doctree b/.doctrees/DEPLOYMENT.doctree new file mode 100644 index 00000000..9b7533ab Binary files /dev/null and b/.doctrees/DEPLOYMENT.doctree differ diff --git a/.doctrees/DETECTORS.doctree b/.doctrees/DETECTORS.doctree new file mode 100644 index 00000000..fdba5656 Binary files /dev/null and b/.doctrees/DETECTORS.doctree differ diff --git a/.doctrees/DETECTOR_GEOMETRY.doctree b/.doctrees/DETECTOR_GEOMETRY.doctree new file mode 100644 index 00000000..d1a27617 Binary files /dev/null and b/.doctrees/DETECTOR_GEOMETRY.doctree differ diff --git a/.doctrees/FPGA.doctree b/.doctrees/FPGA.doctree new file mode 100644 index 00000000..7a50e6fe Binary files /dev/null and b/.doctrees/FPGA.doctree differ diff --git a/.doctrees/FPGA_DATA_ANALYSIS.doctree b/.doctrees/FPGA_DATA_ANALYSIS.doctree new file mode 100644 index 00000000..9a0a9cfe Binary files /dev/null and b/.doctrees/FPGA_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/FPGA_DESIGN.doctree b/.doctrees/FPGA_DESIGN.doctree new file mode 100644 index 00000000..eb7b37e3 Binary files /dev/null and b/.doctrees/FPGA_DESIGN.doctree differ diff --git a/.doctrees/FPGA_LICENSE.doctree b/.doctrees/FPGA_LICENSE.doctree new file mode 100644 index 00000000..9e76d8d4 Binary files /dev/null and b/.doctrees/FPGA_LICENSE.doctree differ diff --git a/.doctrees/FPGA_NETWORK.doctree b/.doctrees/FPGA_NETWORK.doctree new file mode 100644 index 00000000..4fc62c88 Binary files /dev/null and b/.doctrees/FPGA_NETWORK.doctree differ diff --git a/.doctrees/FPGA_PCIE_DRIVER.doctree b/.doctrees/FPGA_PCIE_DRIVER.doctree new file mode 100644 index 00000000..dd8427f5 Binary files /dev/null and b/.doctrees/FPGA_PCIE_DRIVER.doctree differ diff --git a/.doctrees/FPGA_SETTINGS.doctree b/.doctrees/FPGA_SETTINGS.doctree new file mode 100644 index 00000000..f69e27f9 Binary files /dev/null and b/.doctrees/FPGA_SETTINGS.doctree differ diff --git a/.doctrees/HARDWARE.doctree b/.doctrees/HARDWARE.doctree new file mode 100644 index 00000000..e1d2766b Binary files /dev/null and b/.doctrees/HARDWARE.doctree differ diff --git a/.doctrees/HDF5.doctree b/.doctrees/HDF5.doctree new file mode 100644 index 00000000..7e4fa91f Binary files /dev/null and b/.doctrees/HDF5.doctree differ diff --git a/.doctrees/IMAGE_STREAM.doctree b/.doctrees/IMAGE_STREAM.doctree new file mode 100644 index 00000000..52dc4a51 Binary files /dev/null and b/.doctrees/IMAGE_STREAM.doctree differ diff --git a/.doctrees/JFJOCH_BROKER.doctree b/.doctrees/JFJOCH_BROKER.doctree new file mode 100644 index 00000000..811109a8 Binary files /dev/null and b/.doctrees/JFJOCH_BROKER.doctree differ diff --git a/.doctrees/JFJOCH_VIEWER.doctree b/.doctrees/JFJOCH_VIEWER.doctree new file mode 100644 index 00000000..69f06544 Binary files /dev/null and b/.doctrees/JFJOCH_VIEWER.doctree differ diff --git a/.doctrees/JFJOCH_WRITER.doctree b/.doctrees/JFJOCH_WRITER.doctree new file mode 100644 index 00000000..3b872b6c Binary files /dev/null and b/.doctrees/JFJOCH_WRITER.doctree differ diff --git a/.doctrees/LICENSE.doctree b/.doctrees/LICENSE.doctree new file mode 100644 index 00000000..728fbb6b Binary files /dev/null and b/.doctrees/LICENSE.doctree differ diff --git a/.doctrees/NAMING.doctree b/.doctrees/NAMING.doctree new file mode 100644 index 00000000..7badc096 Binary files /dev/null and b/.doctrees/NAMING.doctree differ diff --git a/.doctrees/OPENAPI.doctree b/.doctrees/OPENAPI.doctree new file mode 100644 index 00000000..0bce0219 Binary files /dev/null and b/.doctrees/OPENAPI.doctree differ diff --git a/.doctrees/OPENAPI_SPECS.doctree b/.doctrees/OPENAPI_SPECS.doctree new file mode 100644 index 00000000..2c353cb8 Binary files /dev/null and b/.doctrees/OPENAPI_SPECS.doctree differ diff --git a/.doctrees/PIXEL_MASK.doctree b/.doctrees/PIXEL_MASK.doctree new file mode 100644 index 00000000..4314b21c Binary files /dev/null and b/.doctrees/PIXEL_MASK.doctree differ diff --git a/.doctrees/RELEASE_CONTENTS.doctree b/.doctrees/RELEASE_CONTENTS.doctree new file mode 100644 index 00000000..6e9a9d43 Binary files /dev/null and b/.doctrees/RELEASE_CONTENTS.doctree differ diff --git a/.doctrees/REPOSITORIES.doctree b/.doctrees/REPOSITORIES.doctree new file mode 100644 index 00000000..bc304418 Binary files /dev/null and b/.doctrees/REPOSITORIES.doctree differ diff --git a/.doctrees/RUGNUX.doctree b/.doctrees/RUGNUX.doctree new file mode 100644 index 00000000..c70ab79e Binary files /dev/null and b/.doctrees/RUGNUX.doctree differ diff --git a/.doctrees/SOFTWARE.doctree b/.doctrees/SOFTWARE.doctree new file mode 100644 index 00000000..35d4ddee Binary files /dev/null and b/.doctrees/SOFTWARE.doctree differ diff --git a/.doctrees/SOFTWARE_INTEGRATION.doctree b/.doctrees/SOFTWARE_INTEGRATION.doctree new file mode 100644 index 00000000..9efde916 Binary files /dev/null and b/.doctrees/SOFTWARE_INTEGRATION.doctree differ diff --git a/.doctrees/TESTS.doctree b/.doctrees/TESTS.doctree new file mode 100644 index 00000000..23a3bf61 Binary files /dev/null and b/.doctrees/TESTS.doctree differ diff --git a/.doctrees/THIRD_PARTY_NOTICES.doctree b/.doctrees/THIRD_PARTY_NOTICES.doctree new file mode 100644 index 00000000..dc121364 Binary files /dev/null and b/.doctrees/THIRD_PARTY_NOTICES.doctree differ diff --git a/.doctrees/TOOLS.doctree b/.doctrees/TOOLS.doctree new file mode 100644 index 00000000..ce567a6a Binary files /dev/null and b/.doctrees/TOOLS.doctree differ diff --git a/.doctrees/VERSIONING.doctree b/.doctrees/VERSIONING.doctree new file mode 100644 index 00000000..326131b3 Binary files /dev/null and b/.doctrees/VERSIONING.doctree differ diff --git a/.doctrees/WEB_FRONTEND.doctree b/.doctrees/WEB_FRONTEND.doctree new file mode 100644 index 00000000..c3cefca9 Binary files /dev/null and b/.doctrees/WEB_FRONTEND.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 00000000..b8eea626 Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 00000000..6d325a4b Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/python_client/README.doctree b/.doctrees/python_client/README.doctree new file mode 100644 index 00000000..d631412b Binary files /dev/null and b/.doctrees/python_client/README.doctree differ diff --git a/.doctrees/python_client/docs/AzimIntSettings.doctree b/.doctrees/python_client/docs/AzimIntSettings.doctree new file mode 100644 index 00000000..023070de Binary files /dev/null and b/.doctrees/python_client/docs/AzimIntSettings.doctree differ diff --git a/.doctrees/python_client/docs/BraggIntegrationSettings.doctree b/.doctrees/python_client/docs/BraggIntegrationSettings.doctree new file mode 100644 index 00000000..c3ef6f83 Binary files /dev/null and b/.doctrees/python_client/docs/BraggIntegrationSettings.doctree differ diff --git a/.doctrees/python_client/docs/BrokerStatus.doctree b/.doctrees/python_client/docs/BrokerStatus.doctree new file mode 100644 index 00000000..132fe9db Binary files /dev/null and b/.doctrees/python_client/docs/BrokerStatus.doctree differ diff --git a/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree new file mode 100644 index 00000000..2f30d3f5 Binary files /dev/null and b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree differ diff --git a/.doctrees/python_client/docs/DarkMaskSettings.doctree b/.doctrees/python_client/docs/DarkMaskSettings.doctree new file mode 100644 index 00000000..b7ed23aa Binary files /dev/null and b/.doctrees/python_client/docs/DarkMaskSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettings.doctree b/.doctrees/python_client/docs/DatasetSettings.doctree new file mode 100644 index 00000000..6974dd71 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree new file mode 100644 index 00000000..ef560a43 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree new file mode 100644 index 00000000..1b34cc7b Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree differ diff --git a/.doctrees/python_client/docs/DefaultApi.doctree b/.doctrees/python_client/docs/DefaultApi.doctree new file mode 100644 index 00000000..db30d51b Binary files /dev/null and b/.doctrees/python_client/docs/DefaultApi.doctree differ diff --git a/.doctrees/python_client/docs/Detector.doctree b/.doctrees/python_client/docs/Detector.doctree new file mode 100644 index 00000000..006089ef Binary files /dev/null and b/.doctrees/python_client/docs/Detector.doctree differ diff --git a/.doctrees/python_client/docs/DetectorList.doctree b/.doctrees/python_client/docs/DetectorList.doctree new file mode 100644 index 00000000..f951b0d7 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorList.doctree differ diff --git a/.doctrees/python_client/docs/DetectorListElement.doctree b/.doctrees/python_client/docs/DetectorListElement.doctree new file mode 100644 index 00000000..dda31eb6 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorListElement.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModule.doctree b/.doctrees/python_client/docs/DetectorModule.doctree new file mode 100644 index 00000000..bfdcac79 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModule.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModuleDirection.doctree b/.doctrees/python_client/docs/DetectorModuleDirection.doctree new file mode 100644 index 00000000..b510009f Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModuleDirection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorPowerState.doctree b/.doctrees/python_client/docs/DetectorPowerState.doctree new file mode 100644 index 00000000..115ce332 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorPowerState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSelection.doctree b/.doctrees/python_client/docs/DetectorSelection.doctree new file mode 100644 index 00000000..765b84ce Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSelection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSettings.doctree b/.doctrees/python_client/docs/DetectorSettings.doctree new file mode 100644 index 00000000..2557ede2 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSettings.doctree differ diff --git a/.doctrees/python_client/docs/DetectorState.doctree b/.doctrees/python_client/docs/DetectorState.doctree new file mode 100644 index 00000000..84750d04 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorStatus.doctree b/.doctrees/python_client/docs/DetectorStatus.doctree new file mode 100644 index 00000000..316cb6cb Binary files /dev/null and b/.doctrees/python_client/docs/DetectorStatus.doctree differ diff --git a/.doctrees/python_client/docs/DetectorTiming.doctree b/.doctrees/python_client/docs/DetectorTiming.doctree new file mode 100644 index 00000000..22dc0c03 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorTiming.doctree differ diff --git a/.doctrees/python_client/docs/DetectorType.doctree b/.doctrees/python_client/docs/DetectorType.doctree new file mode 100644 index 00000000..f4e0a6a9 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorType.doctree differ diff --git a/.doctrees/python_client/docs/ErrorMessage.doctree b/.doctrees/python_client/docs/ErrorMessage.doctree new file mode 100644 index 00000000..41404f38 Binary files /dev/null and b/.doctrees/python_client/docs/ErrorMessage.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterFormat.doctree b/.doctrees/python_client/docs/FileWriterFormat.doctree new file mode 100644 index 00000000..42d7d43c Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterFormat.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterSettings.doctree b/.doctrees/python_client/docs/FileWriterSettings.doctree new file mode 100644 index 00000000..5ece661c Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterSettings.doctree differ diff --git a/.doctrees/python_client/docs/FpgaStatusInner.doctree b/.doctrees/python_client/docs/FpgaStatusInner.doctree new file mode 100644 index 00000000..ed1965ab Binary files /dev/null and b/.doctrees/python_client/docs/FpgaStatusInner.doctree differ diff --git a/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree new file mode 100644 index 00000000..3430aa01 Binary files /dev/null and b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/GridScan.doctree b/.doctrees/python_client/docs/GridScan.doctree new file mode 100644 index 00000000..7552bf22 Binary files /dev/null and b/.doctrees/python_client/docs/GridScan.doctree differ diff --git a/.doctrees/python_client/docs/ImageBufferStatus.doctree b/.doctrees/python_client/docs/ImageBufferStatus.doctree new file mode 100644 index 00000000..04efdce5 Binary files /dev/null and b/.doctrees/python_client/docs/ImageBufferStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImageFormatSettings.doctree b/.doctrees/python_client/docs/ImageFormatSettings.doctree new file mode 100644 index 00000000..d795ce4b Binary files /dev/null and b/.doctrees/python_client/docs/ImageFormatSettings.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherStatus.doctree b/.doctrees/python_client/docs/ImagePusherStatus.doctree new file mode 100644 index 00000000..2be4badd Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherType.doctree b/.doctrees/python_client/docs/ImagePusherType.doctree new file mode 100644 index 00000000..b9e53dc3 Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherType.doctree differ diff --git a/.doctrees/python_client/docs/IndexingAlgorithm.doctree b/.doctrees/python_client/docs/IndexingAlgorithm.doctree new file mode 100644 index 00000000..e150689f Binary files /dev/null and b/.doctrees/python_client/docs/IndexingAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/IndexingSettings.doctree b/.doctrees/python_client/docs/IndexingSettings.doctree new file mode 100644 index 00000000..fe809b57 Binary files /dev/null and b/.doctrees/python_client/docs/IndexingSettings.doctree differ diff --git a/.doctrees/python_client/docs/InstrumentMetadata.doctree b/.doctrees/python_client/docs/InstrumentMetadata.doctree new file mode 100644 index 00000000..769d4b73 Binary files /dev/null and b/.doctrees/python_client/docs/InstrumentMetadata.doctree differ diff --git a/.doctrees/python_client/docs/IntegrationModel.doctree b/.doctrees/python_client/docs/IntegrationModel.doctree new file mode 100644 index 00000000..5c3b85f8 Binary files /dev/null and b/.doctrees/python_client/docs/IntegrationModel.doctree differ diff --git a/.doctrees/python_client/docs/JfjochSettings.doctree b/.doctrees/python_client/docs/JfjochSettings.doctree new file mode 100644 index 00000000..0859b393 Binary files /dev/null and b/.doctrees/python_client/docs/JfjochSettings.doctree differ diff --git a/.doctrees/python_client/docs/JfjochStatistics.doctree b/.doctrees/python_client/docs/JfjochStatistics.doctree new file mode 100644 index 00000000..96d2ecff Binary files /dev/null and b/.doctrees/python_client/docs/JfjochStatistics.doctree differ diff --git a/.doctrees/python_client/docs/MeasurementStatistics.doctree b/.doctrees/python_client/docs/MeasurementStatistics.doctree new file mode 100644 index 00000000..a5e56977 Binary files /dev/null and b/.doctrees/python_client/docs/MeasurementStatistics.doctree differ diff --git a/.doctrees/python_client/docs/PcieDevicesInner.doctree b/.doctrees/python_client/docs/PcieDevicesInner.doctree new file mode 100644 index 00000000..909a5bfd Binary files /dev/null and b/.doctrees/python_client/docs/PcieDevicesInner.doctree differ diff --git a/.doctrees/python_client/docs/PixelMaskStatistics.doctree b/.doctrees/python_client/docs/PixelMaskStatistics.doctree new file mode 100644 index 00000000..d3e9f09d Binary files /dev/null and b/.doctrees/python_client/docs/PixelMaskStatistics.doctree differ diff --git a/.doctrees/python_client/docs/Plot.doctree b/.doctrees/python_client/docs/Plot.doctree new file mode 100644 index 00000000..b9bb9578 Binary files /dev/null and b/.doctrees/python_client/docs/Plot.doctree differ diff --git a/.doctrees/python_client/docs/PlotUnitX.doctree b/.doctrees/python_client/docs/PlotUnitX.doctree new file mode 100644 index 00000000..5f0ace53 Binary files /dev/null and b/.doctrees/python_client/docs/PlotUnitX.doctree differ diff --git a/.doctrees/python_client/docs/Plots.doctree b/.doctrees/python_client/docs/Plots.doctree new file mode 100644 index 00000000..299ab240 Binary files /dev/null and b/.doctrees/python_client/docs/Plots.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimList.doctree b/.doctrees/python_client/docs/RoiAzimList.doctree new file mode 100644 index 00000000..f9a57907 Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimList.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimuthal.doctree b/.doctrees/python_client/docs/RoiAzimuthal.doctree new file mode 100644 index 00000000..9f8e9084 Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimuthal.doctree differ diff --git a/.doctrees/python_client/docs/RoiBox.doctree b/.doctrees/python_client/docs/RoiBox.doctree new file mode 100644 index 00000000..929be63d Binary files /dev/null and b/.doctrees/python_client/docs/RoiBox.doctree differ diff --git a/.doctrees/python_client/docs/RoiBoxList.doctree b/.doctrees/python_client/docs/RoiBoxList.doctree new file mode 100644 index 00000000..65cdc1ef Binary files /dev/null and b/.doctrees/python_client/docs/RoiBoxList.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircle.doctree b/.doctrees/python_client/docs/RoiCircle.doctree new file mode 100644 index 00000000..3fd5dacf Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircle.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircleList.doctree b/.doctrees/python_client/docs/RoiCircleList.doctree new file mode 100644 index 00000000..ea9ddcaf Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircleList.doctree differ diff --git a/.doctrees/python_client/docs/RoiDefinitions.doctree b/.doctrees/python_client/docs/RoiDefinitions.doctree new file mode 100644 index 00000000..e282ae2c Binary files /dev/null and b/.doctrees/python_client/docs/RoiDefinitions.doctree differ diff --git a/.doctrees/python_client/docs/RotationAxis.doctree b/.doctrees/python_client/docs/RotationAxis.doctree new file mode 100644 index 00000000..17b00e70 Binary files /dev/null and b/.doctrees/python_client/docs/RotationAxis.doctree differ diff --git a/.doctrees/python_client/docs/ScanResult.doctree b/.doctrees/python_client/docs/ScanResult.doctree new file mode 100644 index 00000000..c6b20533 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResult.doctree differ diff --git a/.doctrees/python_client/docs/ScanResultImagesInner.doctree b/.doctrees/python_client/docs/ScanResultImagesInner.doctree new file mode 100644 index 00000000..d6e83392 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResultImagesInner.doctree differ diff --git a/.doctrees/python_client/docs/SpotFindingSettings.doctree b/.doctrees/python_client/docs/SpotFindingSettings.doctree new file mode 100644 index 00000000..d8e56788 Binary files /dev/null and b/.doctrees/python_client/docs/SpotFindingSettings.doctree differ diff --git a/.doctrees/python_client/docs/StandardDetectorGeometry.doctree b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree new file mode 100644 index 00000000..04e06fea Binary files /dev/null and b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree differ diff --git a/.doctrees/python_client/docs/TcpSettings.doctree b/.doctrees/python_client/docs/TcpSettings.doctree new file mode 100644 index 00000000..355545e8 Binary files /dev/null and b/.doctrees/python_client/docs/TcpSettings.doctree differ diff --git a/.doctrees/python_client/docs/UnitCell.doctree b/.doctrees/python_client/docs/UnitCell.doctree new file mode 100644 index 00000000..536658f2 Binary files /dev/null and b/.doctrees/python_client/docs/UnitCell.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree new file mode 100644 index 00000000..6af290dc Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree new file mode 100644 index 00000000..cbd200c2 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqSettings.doctree b/.doctrees/python_client/docs/ZeromqSettings.doctree new file mode 100644 index 00000000..633af6f5 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqSettings.doctree differ diff --git a/ACKNOWLEDGEMENT.html b/ACKNOWLEDGEMENT.html new file mode 100644 index 00000000..9158aed3 --- /dev/null +++ b/ACKNOWLEDGEMENT.html @@ -0,0 +1 @@ + Acknowledgements — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Acknowledgements

Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang “Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors” (2023), J. Synchrotron Rad., 30, 227-234 doi:10.1107/S1600577522010268.

The project is supported by :

  • Innosuisse via Innovation Project “NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications” (101.535.1 IP-ENG; Apr 2023 - Sep 2025).

  • ETH Domain via Open Research Data Contribute project (Jan - Dec 2023)

  • AMD University Program with donation of licenses of Ethernet IP cores and Vivado software

Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading the result, follows Jon Wright (ESRF): “Experiences with GPU decompression for bitshuffle + LZ4 data”, HDF5 User Group meeting (2021), and bslz4decoders. The CUDA kernels in Jungfraujoch are its own, but the approach is his.

Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, A. Salzburger, S. N. Swatman and B. Yeo, “traccc: GPU track reconstruction library for HEP experiments” (2025), arXiv:2505.22822; traccc. The CPU spot extractor adapts its SparseCCL source, and the CUDA spot extractor follows the design of its GPU counterpart - a backward-neighbour graph over a sorted hit list, resolved by a parallel union-find. traccc is MPL-2.0; see THIRD_PARTY_NOTICES.md.

This software uses Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license

Crystallographic methods adopted from other packages

The analysis pipeline reimplements methods first published, and in most cases first implemented, by other crystallographic software. The code below is Jungfraujoch’s own; the methods are theirs, and are acknowledged here. Where a package’s source was consulted this is said explicitly. None of these packages is linked or vendored, with the single exception of GEMMI (see THIRD_PARTY_NOTICES.md).

XDS — rotation geometry and notation, the reciprocal Lorentz and partiality treatment, the maximum-likelihood mosaicity estimate, the MINPK criterion for rejecting a reflection whose predicted profile is not cleanly its own, the intensity-based test for a centred lattice, and the scaling correction surfaces indexed by image number and detector region. W. Kabsch, “XDS” (2010), Acta Cryst. D66, 125-132 doi:10.1107/S0907444909047337; W. Kabsch, “Integration, scaling, space-group assignment and post-refinement” (2010), Acta Cryst. D66, 133-144 doi:10.1107/S0907444909047374.

Profile fitting with reweighted, de-biased variances is the Kabsch/Otwinowski iteration, from the second XDS paper above and from Z. Otwinowski and W. Minor, “Processing of X-ray diffraction data collected in oscillation mode” (1997), Methods Enzymol. 276, 307-326 doi:10.1016/S0076-6879(97)76066-X.

DIALS — the resolution cutoff from the CC1/2 fall-off, per-observation outlier rejection at merge, the scaling error model, and the treatment of a reflection whose background is contaminated. Its published behaviour, and in places its source, settled several choices here. G. Winter, D. G. Waterman, J. M. Parkhurst et al., “DIALS: implementation and evaluation of a new integration package” (2018), Acta Cryst. D74, 85-97 doi:10.1107/S2059798317017235; D. G. Waterman, G. Winter, R. J. Gildea et al., “Diffraction-geometry refinement in the DIALS framework” (2016), Acta Cryst. D72, 558-575 doi:10.1107/S2059798316002187; J. Beilsten-Edmands, G. Winter, R. Gildea et al., “Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling” (2020), Acta Cryst. D76, 385-399 doi:10.1107/S2059798320003198; J. M. Parkhurst, G. Winter, D. G. Waterman et al., “Robust background modelling in DIALS” (2016), J. Appl. Cryst. 49, 1912-1921 doi:10.1107/S1600576716013595.

POINTLESS (CCP4) — the space-group search. Stage A scores each candidate rotation operator by the correlation of I(h) with I(Rh); the screw-axis test scores a predicted-absent class against the rest of its own axial row rather than against a global mean or a fixed cut, and lets confidence fall away with the number of axial reflections instead of refusing below a count. P. Evans, “Scaling and assessment of data quality” (2006), Acta Cryst. D62, 72-82 doi:10.1107/S0907444905036693; P. R. Evans, “An introduction to data reduction: space-group determination, scaling and intensity statistics” (2011), Acta Cryst. D67, 282-292 doi:10.1107/S090744491003982X; P. R. Evans and G. N. Murshudov, “How good are my data and what is the resolution?” (2013), Acta Cryst. D69, 1204-1214 doi:10.1107/S0907444913000061; J. Agirre, M. Atanasova, H. Bagdonas et al., “The CCP4 suite: integrative software for macromolecular crystallography” (2023), Acta Cryst. D79, 449-461 doi:10.1107/S2059798323003595.

MOSFLM — the Rossmann FFT autoindexing algorithm and post-refinement practice, including which parameters are safe to refine per image and which must be refined over a wedge. A. G. W. Leslie and H. R. Powell, “Processing diffraction data with MOSFLM” (2007), in Evolving Methods for Macromolecular Crystallography, NATO Science Series II, vol. 245, 41-51 doi:10.1007/978-1-4020-6316-9_4; T. G. G. Battye, L. Kontogiannis, O. Johnson, H. R. Powell and A. G. W. Leslie, “iMOSFLM: a new graphical interface for diffraction-image processing with MOSFLM” (2011), Acta Cryst. D67, 271-281 doi:10.1107/S0907444910048675; H. R. Powell, T. G. G. Battye, L. Kontogiannis, O. Johnson and A. G. W. Leslie, “Integrating macromolecular X-ray diffraction data with the graphical user interface iMosflm” (2017), Nat. Protoc. 12, 1310-1325 doi:10.1038/nprot.2017.037.

CrystFEL — spot finding, the three-ring integration region, the serial/stills processing model, and the per-frame indexing acceptance test (indexing_peak_check() in peaks.c). T. A. White, R. A. Kirian, A. V. Martin, A. Aquila, K. Nass, A. Barty and H. N. Chapman, “CrystFEL: a software suite for snapshot serial crystallography” (2012), J. Appl. Cryst. 45, 335-341 doi:10.1107/S0021889812002312.

GEMMI — symmetry operations, unit-cell and structure-factor machinery, and MTZ / XDS_ASCII I/O. Vendored in gemmi_gph/, so it also carries a licence obligation. M. Wojdyr, “GEMMI: A library for structural biology” (2022), J. Open Source Softw. 7, 4200 doi:10.21105/joss.04200.

Data-quality statistics follow the established conventions rather than any one program: R_meas and R_pim, CC1/2 and CC*, and the reporting of I/sigma(I). K. Diederichs and P. A. Karplus, “Improved R-factors for diffraction data analysis in macromolecular crystallography” (1997), Nat. Struct. Biol. 4, 269-275 doi:10.1038/nsb0497-269; P. A. Karplus and K. Diederichs, “Linking crystallographic model and data quality” (2012), Science 336, 1030-1033 doi:10.1126/science.1218231; K. Diederichs and P. A. Karplus, “Better models by discarding data?” (2013), Acta Cryst. D69, 1215-1222 doi:10.1107/S0907444913001121.

Uncertainty conventions follow the IUCr Commission on Crystallographic Nomenclature: D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., “Statistical descriptors in crystallography: Report of the IUCr Subcommittee on Statistical Descriptors” (1989), Acta Cryst. A45, 63-75 doi:10.1107/S0108767388009596; D. Schwarzenbach, S. C. Abrahams, H. D. Flack, E. Prince and A. J. C. Wilson, “Statistical descriptors in crystallography. II. Report of a Working Group on Expression of Uncertainty in Measurement” (1995), Acta Cryst. A51, 565-569 doi:10.1107/S0108767395002340.

\ No newline at end of file diff --git a/CBOR.html b/CBOR.html new file mode 100644 index 00000000..f7a4026a --- /dev/null +++ b/CBOR.html @@ -0,0 +1 @@ + CBOR messages — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

CBOR messages

To communicate between FPGA-equipped receiver system and writers, Jungfraujoch is using binary CBOR encoding with tinycbor library (Intel). The protocol is based on and compatible with DECTRIS Stream2. There are minor differences at the moment:

  • LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed

  • Few fields are currently absent

  • Extra fields are present beyond DECTRIS standard

  • There are calibration and metadata messages defined beyond DECTRIS specification

Start message

Field name

Type

Description

Present in DECTRIS format

type

String

value “start”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

detector_distance

float

Detector distance [m]

detector_translation

Array(float)

Detector translation vector [m]

X

beam_center_x

float

Beam center in X direction [pixels]

X

beam_center_y

float

Beam center in Y direction [pixels]

X

countrate_correction_enabled

bool

Countrate correction enabled

X

flatfield_enabled

bool

Flatfield enabled

X

number_of_images

uint64

Number of images in the series

X

image_size_x

uint64

Image width [pixels]

X

image_size_y

uint64

Image height [pixels]

X

incident_energy

float

X-ray energy [eV]

X

incident_wavelength

float

X-ray wavelength [Angstrom]

X

incident_wavelength_spread

float (optional)

FWHM of the X-ray wavelength distribution [Angstrom] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic

frame_time

float

Frame time, if multiple frames per trigger [s]

X

count_time

float

Exposure time [s]

X

saturation_value

int64

Maximum valid sample value

X

error_value

int64 (optional)

Value used in images to describe pixels that are in error state or missing

pixel_size_x

float

Pixel width [m]

X

pixel_size_y

float

Pixel height [m]

X

sensor_thickness

float

Sensor thickness [m]

X

sensor_material

string

Sensor material

X

arm_date

date

Approximate date of arming

X

pixel_mask_enabled

bool

Pixel mask applied on images

X

detector_description

string

Name of the detector

X

detector_serial_number

string

Detector serial number

X

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

fluorescence

object (optional)

X-ray fluorescence spectrum collected at start

- energy

Array(float)

Energy of measuring point [eV]

- data

Array(float)

Fluorescence scan result data [arbitrary units]; must be strictly the same length as energy

goniometer

Map

Definition of rotation axis (optional)

X

- AXIS

string

Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch

X

- - increment

float

Rotation axis increment (per image) in degree [deg]

X

- - start

float

Rotation axis start angle [deg]

X

- - axis

Array(float)

Vector for the rotation axis

- - helical_step

Array(float)

Translation for helical scan for 1 image [m]

- - screening_wedge

Array(float)

Wedge for screening [deg] (increment would correspond to difference between screening points)

grid_scan

object

Grid scan definition (optional and exclusive with rotation axis)

- n_fast

uint64

Number of elements along fast axis

- n_slow

uint64

Number of elements along slow axis

- step_x_axis

float

Step along X axis, can be negative [m]

- step_y_axis

float

Step along Y axis, can be negative [m]

- snake_scan

bool

Snake scan (rows alternate direction)

- vertical_scan

bool

Vertical scan (enabled: fast direction = Y, disabled: fast direction = X)

jungfrau_conversion_enabled

bool (optional)

Applying JUNGFRAU pixel conversion (to photons or keV)

jungfrau_conversion_factor

float (optional)

Factor used for JUNGFRAU conversion [eV]

geometry_transformation_enabled

bool (optional)

Transformation from detector module geometry (512x1024) to full detector geometry

pixel_mask

Map(string -> Image)

Pixel mask - multiple in case of storage cells

X

channels

Array(string)

List of image channels

X

max_spot_count

uint64

Maximum number of spots identified in spot finding

max_extra_lattices

uint64

Maximum number of extra lattices

storage_cell_number

uint64 (optional)

Number of storage cells used by JUNGFRAU

storage_cell_delay

Rational

Delay of storage cells in JUNGFRAU

threshold_energy

Map(string -> float)

Per-channel threshold energy [eV] (map of channel name to value)

image_dtype

string

Pixel bit type (e.g. uint16)

X

unit_cell

object (optional)

Unit cell of the system: a, b, c [angstrom] and alpha, beta, gamma [degree]

az_int_q_bin_count

uint64

Number of azimuthal integration bins in the radial direction

az_int_phi_bin_count

uint64

Number of azimuthal integration bins in the phi angle direction

az_int_bin_to_q

Array(float)

Q value for each azimuthal integration bin [angstrom^-1]

az_int_bin_to_two_theta

Array(float)

Two theta angle value for each azimuthal integration bin [deg]

az_int_bin_to_phi

Array(float)

Phi value for each azimuthal integration bin [deg]

az_int_map

Image

Mapping between pixel and bin number

summation

uint64

Factor of frame summation

user_data

string

JSON serialized to string that can contain the following fields (all fields are optional):

X

- file_prefix

string

File prefix

- images_per_file

uint64

Number of images written per file

- images_per_trigger

uint64

Number of images collected per trigger

- source_name

string

Facility name

- source_type

string

Type of X-ray source (use NXsource/type values, for example “Synchrotron X-ray Source” or “Free-Electron Laser”)

- instrument_name

string

Instrument name

- sample_name

string

Name of the sample

- user

any valid JSON

Value of header_appendix provided at collection start to Jungfraujoch

- attenuator_transmission

float

Attenuator transmission []

- total_flux

float

Total flux [ph/s]

- space_group_number

uint64

Space group number

- summation_mode

string

Summation mode (internal|fpga|cpu)

- overwrite

bool

Overwrite existing HDF5 files

- file_format

int

File writer format: 0 = only data files, 1 = NXmx legacy soft links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF, 5 = TIFF, 6 = no file written

- roi

Array(object)

ROI configurations; each element is one of:

type “box”: xmin, xmax, ymin, ymax (numbers)

type “circle”: r, x, y (numbers)

type “azim”: qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector

- gain_file_names

Array(string)

Names of JUNGFRAU gain files used for the current detector

- write_master_file

bool

With multiple sockets, it selects which socket will provide master file

- write_images

bool

Write images in the HDF5 file (if false, will only write metadata)

- data_reduction_factor_serialmx

uint64

Data reduction factor for serial MX

- experiment_group

string

ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number

- jfjoch_release

string

Jungfraujoch release number

- socket_number

uint64

Number of ZeroMQ socket (on jfjoch_broker side) used for transmission

- bit_depth_readout

uint64

Bit depth of the detector readout

- writer_notification_zmq_addr

string

ZeroMQ address to inform jfjoch_broker about writers that finished operation

- xfel_pulse_id

uint64

Pulse IDs are recorded for images

- ring_current_mA

float

Ring current at the start of the measurement

- sample_temperature_K

float

Sample temperature [K]

- detect_ice_rings

bool

Ice ring detection feature is enabled

- indexing_algorithm

string

Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none

- geom_refinement_algorithm

string

Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center

- poni_rot1

float

Tilt of the detector rot1 according to PyFAI PONI convention [rad]

- poni_rot2

float

Tilt of the detector rot2 according to PyFAI PONI convention [rad]

- poni_rot3

float

Tilt of the detector rot3 according to PyFAI PONI convention [rad]

See DECTRIS documentation for definition of Image as MultiDimArray with optional compression.

Image message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “image”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

real_time

Rational

Exposure time

X

start_time

Rational

Exposure start time (highly approximate)

X

end_time

Rational

Exposure end time (highly approximate)

X

spots

Array(object)

Spots:

- x

float

observed position in x (pixels)

- y

float

observed position in y (pixels)

- I

float

intensity (photons)

- maxc

int64

max count (photons)

- ice_ring

bool

spot in resolution range for ice rings

- indexed

bool

indexed solution

- latt

int64

Lattice to which the peak belongs (negative number = not indexed)

- image

int64

image number the spot belongs to

- h

int64

Miller index (indexed spots only)

- k

int64

Miller index (indexed spots only)

- l

int64

Miller index (indexed spots only)

- dist_ewald

float

distance to Ewald sphere [Angstrom^-1] (indexed spots only)

reflections

Array(object)

Reflections:

- h

int64

Miller index

- k

int64

Miller index

- l

int64

Miller index

- x

float

prediced position in x (pixels)

- y

float

predicted position in y (pixels)

- obs_x

float

observed position in x (pixels)

- obs_y

float

observed position in y (pixels)

- d

float

resolution [Angstrom]

- I

float

integrated intensity (photons)

- bkg

float

mean background value (photons)

- var_bkg

float

non-signal (background) part of sigma^2, carried to the merge (photons^2)

- sigma

float

standard deviation, estimated from counting statistics (photons)

- image

float

image number (present for each spot)

- rp

float

Distance to Ewald sphere [Angstrom^-1]

- rlp

float

Reciprocal Lorentz and polarization corrections

- partiality

float

Partiality of the reflection

- phi

float

phi angle from XDS: difference from middle of current frame, not absolute [deg]

- zeta

float

Lorentz zeta factor (reciprocal-space geometry term)

- image_scale_corr

float

Per-image scale correction; I_true = image_scale_corr * I

spot_count

uint64

Spot count

spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

spot_count_low_res

uint64

Number of spots in low resolution (prior to filtering)

spot_count_indexed

uint64

Number of spots which fit indexing solution within a given tolerance

az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

NaN is used for empty bins and has to be taken care by the receiver

az_int_profile_std

Array(float)

Standard deviation for azimuthal integration. (NaN for less than 2 samples)

az_int_profile_count

Array(uint64)

Number of pixels contributing to azimuthal bin

indexing_result

bool

Indexing successful

indexing_lattice_count

int64

Number of indexing lattices found for this image

indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

indexing_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

profile_radius

float

Profile radius of the image - describes distance of observed reflections from the Ewald sphere [Angstrom^-1]

integrated_reflections

int64

Count of integrated reflections

mosaicity

float

Angular range of spots in image from a rotation scan [degree]

b_factor

float

Estimated B-factor (Angstrom^2)

compression_time

float

Time spent on compression/decompressing image [s]

preprocessing_time

float

Time spent on preparing the image for analysis [s]

azint_time

float

Time spent on azimuthal integration [s]

spot_finding_time

float

Time spent on spot finding [s]

indexing_time

float

Time spent on indexing [s]

refinement_time

float

Time spent on refinement of indexing solution and experimental geometry [s]

index_analysis_time

float

Time spent on analyzing idnexing solution, calculating profile radius and mosaicity [s]

bragg_prediction_time

float

Time spent on predicting Bragg spots [s]

integration_time

float

Time spent on Bragg integration [s]

image_scale_time

float

Time spent on on-the-fly scaling [s]

processing_time

float

Total processing time [s]

xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

lattice_type

object

Bravais lattice classification of the indexing result (present only if available)

X

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

jf_info

uint64

Detector info field

receiver_aq_dev_delay

uint64

Receiver internal delay

receiver_free_send_buf

uint64

Receiver internal number of available buffer locations

receiver_buf_in_sending

uint64

Receiver internal number of buffer locations currently in sending/writing

receiver_buf_in_preparation

uint64

Receiver internal number of buffer locations currently in processing

storage_cell

uint64

Storage cell number

saturated_pixel_count

uint64

Saturated pixel count

pixel_sum

uint64

Sum of all pixels, excl. error and saturation

error_pixel_count

uint64

Error pixel count

strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

min_viable_pixel_value

int64

Minimal pixel value, excl. error and saturation

max_viable_pixel_value

int64

Maximal pixel value, excl. error and saturation

resolution_estimate

float

Diffraction resolution estimation [Angstrom]

X

data_collection_efficiency

float

Image collection efficiency []

packets_expected

uint64

Number of packets expected per image (in units of 2 kB)

packets_received

uint64

Number of packets received per image (in units of 2 kB)

bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons]

ice_ring_score

float

Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

spot_count_ice_control

float

Spots in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands’ own q width (control for spot_count_ice_rings)

beam_corr_x

float

Beam center correction X applied during processing [pixel]

X

beam_corr_y

float

Beam center correction Y applied during processing [pixel]

X

image_scale_factor

float

Scaling result: Image scale factor (g)

X

image_scale_mosaicity

float

Scaling result: Image scale mosaicity [deg]

X

image_scale_cc

float

Scaling result: Image scale CC

X

adu_histogram

Array(uint64)

ADU histogram

roi_integrals

object

Results of ROI calculation

X

- sum

int64

Sum of pixels in ROI area [photons]

- sum_square

int64

Sum of squares of pixels in ROI area [photons]

- pixels

uint64

Valid pixels in ROI area

- max_count

int64

Highest count in ROI area [photons]

- x_weighted_sum

int64

ROI pixel X position multiplied by photon count [photons * pixels]

- y_weighted_sum

int64

ROI pixel Y position multiplied by photon count [photons * pixels]

user_data

string

Optional user defined text information - this is image_appendix serialized to JSON format

X

data

Map(string -> Image)

Image

X

Metadata message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “metadata”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

images

Array(object)

Array of images (order and size of the array are not guaranteed)

X

- image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

- original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

- real_time

Rational

Exposure time

X

- start_time

Rational

Exposure start time (highly approximate)

X

- end_time

Rational

Exposure end time (highly approximate)

X

- spot_count

uint64

Spot count

- spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

- az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

- indexing_result

bool

Indexing successful

- indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

- indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

- xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

- xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

- jf_info

uint64

Detector info field

- receiver_aq_dev_delay

uint64

Receiver internal delay

- receiver_free_send_buf

uint64

Receiver internal number of available send buffers

- storage_cell

uint64

Storage cell number

- saturated_pixel_count

uint64

Saturated pixel count

- error_pixel_count

uint64

Error pixel count

- strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

- data_collection_efficiency

float

Image collection efficiency []

- bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons] (with solid angle/polarization corrections, if applied)

X

- resolution_estimate

float

Diffraction resolution estimation

X

- adu_histogram

Array(uint64)

ADU histogram

X

- roi_integrals

object

Results of ROI calculation

X

- - sum

int64

Sum of pixels in ROI area [photons]

- - sum_square

int64

Sum of squares of pixels in ROI area [photons]

- - pixels

uint64

Valid pixels in ROI area

- - max_count

int64

Highest count in ROI area [photons]

End message

Field name

Type

Description

Present in DECTRIS format

type

String

value “end”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

end_date

string

Approximate end date

max_image_number

uint64

Number of image with the highest number; counted from 1 to distinguish zero images and one image

images_collected

uint64

Number of images collected

images_sent_to_write

uint64

Number of images sent to writer; if writer queues were full, it is possible this is less than images collected

data_collection_efficiency

float

Overall network packets collected / network packets expected

az_int_result

Map(text->Array(float))

Azimuthal integration results, use az_int_bin_to_q from start message for legend

adu_histogram

Map(text->Array(uint64))

ADU values histogram

adu_histogram_bin_width

uint64

Width of bins in the above histogram [ADU]

max_receiver_delay

uint64

Internal performance of Jungfraujoch

bkg_estimate

float

Mean background estimate for the whole run

indexing_rate

float

Mean indexing rate for the whole run

unit_cell

object (optional)

Unit cell of the system, based on the actual experiment: a, b, c [angstrom] and alpha, beta, gamma [degree]

rotation_lattice_type

object

Bravais lattice classification of the total rotation solution over the run, if available; same schema as lattice_type

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

rotation_lattice

Array(9 * float)

Real-space lattice basis, flattened 3x3 in row-major order

rotation_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

data_collection_efficiency_image

Array(float)

Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context

spot_count

Array(int32)

Per-image spot count

spot_count_ice_ring

Array(int32)

Per-image number of spots within identified ice-ring resolution ranges

spot_count_low_res

Array(int32)

Per-image number of low-resolution spots

spot_count_indexed

Array(int32)

Per-image number of spots fitting indexing solution

image_indexed

Array(uint8)

Per-image indexing result; 0 = not indexed, nonzero = indexed

v_bkg_estimate

Array(float)

Per-image background estimate

ice_ring_score

Array(float)

Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice)

spot_count_ice_control

Array(float)

Per-image spot count in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands’ q width

ice_ring_score_mean

float

Mean ice-ring score for the whole run (1 = no ice)

profile_radius

Array(float)

Per-image profile radius [Angstrom^-1]

mosaicity

Array(float)

Per-image mosaicity [degree]

bFactor

Array(float)

Per-image estimated B-factor [Angstrom^2]

resolution_estimate

Array(float)

Per-image diffraction resolution estimate [Angstrom]

min_viable_pixel_value

Array(int64)

Per-image minimum valid pixel value, excluding error/saturated pixels

max_viable_pixel_value

Array(int64)

Per-image maximum valid pixel value, excluding error/saturated pixels

saturated_pixel_count

Array(int32)

Per-image saturated pixel count

error_pixel_count

Array(int32)

Per-image error pixel count

image_scale_factor

Array(float)

Per-image scale factor, if scaling/merging was performed

integrated_reflections

Array(int32)

Per-image count of integrated reflections

indexed_lattice_count

Array(int32)

Per-image count of indexed lattices

niggli_class

Array(uint8)

Per-image Niggli class identifier for indexed images; 0 if unavailable

pixel_sum

Array(int64)

Per-image sum of all valid pixels, excluding error/saturated pixels

image_scale_mosaicity

Array(float)

Scaling result: Image scale mosaicity [deg]

image_scale_cc

Array(float)

Scaling result: Image scale CC

End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted.

Calibration message

Field name

Type

Description

Present in DECTRIS format

type

String

value “calibration”

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

data

Map(string -> Image)

Calibration map (only single pedestal array per message)

User data

In many cases there is an interest from facilities to forward more metadata, than available explicitly in the Jungfraujoch. For this reason two fields can be provided: header_appendix (sent with start message) and image_appendix (send with image message). To increase flexibility, both appendices can contain any valid JSON message. These appendices are serialized into string and stored in CBOR messages as user_data.

Notably for start message, user_data can contain more information (non-DECTRIS compliant metadata). Therefore user_data is serialized by Jungfraujoch as CBOR object. There is member user which contains header_appendix defined in OpenAPI of Jungfraujoch.

Notes on images and compression

  • Images are encoded as DECTRIS MultiDimArray with typed array tags:

    • For RGB: shape [3, height, width], type: u8

    • For grayscale: shape [height, width], type according to bit depth and sign (e.g., uint16 LE)

  • Compression:

    • Uncompressed: raw CBOR byte string

    • Bitshuffle+LZ4: tag with [“bslz4”, elem_size, bytes]

    • Bitshuffle+Zstandard: tag with [“bszstd”, elem_size, bytes]

Notes on typed arrays

Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays.

Common tags used in this protocol include:

  • float32 little-endian arrays for Array(float)

  • uint8 arrays for compact boolean/integer flags such as image_indexed

  • int32 little-endian arrays for per-image counts

  • int64 little-endian arrays for large per-image integer values

  • uint64 little-endian arrays for histograms

\ No newline at end of file diff --git a/CHANGELOG.html b/CHANGELOG.html new file mode 100644 index 00000000..4c3d560d --- /dev/null +++ b/CHANGELOG.html @@ -0,0 +1 @@ + Changelog — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Changelog

1.0.0

1.0.0-rc.161

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux: significantly better quality of results, and faster. A large rework of integration, scaling, merging, geometry refinement and space-group determination, together with measurements the program previously made no attempt at - the direct beam before indexing, the beam stop, the goniometer rotation scale, and the stretches of a sweep the crystal did not deliver. A rotation dataset typically gains observations at better <I/sigma> and R_meas, and every mx and scale run writes a <prefix>_report.txt results report modelled on XDS’s CORRECT.LP. Many defaults moved with it: spot detection is self-calibrating, beam-stop detection and rotation geometry post-refinement are on, resolution limits default to as far as the detector reaches, and ice-ring handling engages only where the crystal is measured to have ice.

  • jfjoch_viewer: the beam-stop shadow, the detector calibration and the beam-centre measurement are reachable from “Analyze dataset”; the settings panel reports how the sample moved and how polarized the beam was; image rendering and interaction are faster.

  • Performance: bitshuffle+LZ4 images are decoded on the GPU rather than on the host, with the bitshuffle inverse fused into preprocessing so the decompressed frame is never held in device memory.

  • Broker, writer, packaging and build: image-slot lifetime and locking fixes, per-image datasets sized by the images actually written, the Debian/Ubuntu broker package renamed to jfjoch, and image_analysis compiling under MSVC again.

Breaking change to the rugnux command line:

  • --azint-only and --scale are removed, replaced by --mode azint and --mode scale; the full pipeline is --mode mx and remains the default. A script passing the old flags now fails with the list of valid modes rather than silently running the wrong one.

  • -t/--stride is refused on rotation data: skipping frames cuts every reflection’s rocking curve, so the combined fulls and their partiality would be measured over frames the sweep never recorded. Select a contiguous range with -s/-e instead. --mode azint and --force-still still take a stride.

Breaking changes to OpenAPI - regenerate the client (jfjoch-client 1.0.0-rc.161, frontend/src/client) or read the affected fields as optional:

  • image_scale_b is removed from the plot_type enum, so a client requesting that plot now gets an error rather than a curve.

  • azim_int_settings.high_q_recipA, spot_finding_settings.high_resolution_limit and spot_finding_settings.low_resolution_limit are no longer required. All three mean “no limit at that end” when unset and are omitted from the response instead of carrying a placeholder value, which raises in a client generated from an rc.160-or-earlier spec. A value of 0 is still accepted and means the same thing.

Breaking changes to the stored formats - a consumer reading these fields must treat them as optional:

  • The per-image image-scale B factor is no longer computed, so /entry/MX/imageScaleBFactor is absent from newly written HDF5 files and the corresponding key is absent from the CBOR DataMessage and END blocks. Files written by rc.160 and earlier still contain it and still open; nothing in the pipeline reads it any more.

  • _reflns.jfjoch_diffrn_ISa now carries the whole-range 1/sqrt(a*b) that XDS’s ISa denotes, and the error-model a and b are reported in XDS’s convention; the strong-reflection asymptote moves to _reflns.jfjoch_diffrn_ISa_asymptotic. A file written by an earlier version carries the asymptote under the plain ISa name.

1.0.0-rc.160

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux: Rotation geometry post-refinement is now on by default (--rotation-no-postrefine to disable; also a viewer checkbox). A first pass integrates at the header geometry, then the shared detector distance + beam centre and the crystal cell/goniometer-axis are post-refined over all frames (cross-validated, committed only for a small < 1 % move, with the gauge-weak beam centre restrained toward the header); a second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical <prefix>_* output; the header-geometry pass is kept as <prefix>_01_*.

  • rugnux: Optional per-batch relative-B correction for rotation (--relative-b[=deg], default 10°-of-rotation batches when bare, off otherwise) - a cross-validated, curvature-smoothed resolution×dose correction beyond the single global decay slope.

  • rugnux: Always-on radiation-damage report for rotation - the per-image scale correlation-to-merge and mosaicity versus dose, plus the relative B-factor change over the run (first→last) as a scalar and a per-batch relative-B curve, printed to the log and written to the merged mmCIF. Report-only; it never alters the merge.

  • rugnux: De-novo space-group search ranks candidate lattice centerings by net absences (systematically-absent minus violating), not the gross absent count, fixing an over-centering of a genuinely C-centred lattice to F.

  • rugnux: Record the producing software (name and version) and the refined detector distance and beam centre in the merged mmCIF (and the software in the MTZ history).

  • Bragg integration: Carry the box-sum observed centroid through the profile-fit path, so the observed spot centroid is emitted in every integrator mode.

  • rugnux: Report ISa as the counting-subtracted strong-reflection asymptote, not 1/b of the whole-range fit; it also sets the merged-sigma floor. CC1/2, R-meas and per-obs sigmas unchanged.

  • Frontend: Azimuthal-integration Q fields (Q spacing / Low Q / High Q) accept 5 decimals (was 3), matching the 1e-5 q_spacing minimum; number-field precision is now configurable.

  • rugnux: Add a dataset-wide Wilson B-factor estimate to the merged output (mmCIF, stats table, log); the per-image viewer Wilson B emits NaN for implausible fits.

  • rugnux: De-novo space-group search vetoes a merohedral-twin over-promotion whose systematic error-model b balloons past a calibrated bound (keeps R3 as R3, not R32).

  • rugnux: De-novo space-group search decides lattice centering from the strength (mean I/sigma) of the systematically-absent class, not a per-reflection violation count.

  • rugnux: Recover lattice centering on weak / low-energy data via a floor-independent test (rate of significant absent vs present reflections), fixing a missed I-centring at 5/13 keV.

  • rugnux: Report anomalous signal-to-noise SigAno = <|I(+)-I(-)|>/ per shell and overall (mmCIF PDBx items + stats-table column); anomalous merges only.

  • rugnux: De-novo space-group search recovers a genuine high-symmetry group on weak data with a broken sigma model by confirming on the systematic-b test alone (restores an F432 case).

  • rugnux: Print the adopted space group and unit cell as a one-line summary at the end of the run (de-novo or user-fixed -S).

  • rugnux: Score the radiation-damage decay cross-validation on a sigma-independent (R-meas-like) metric, so a spurious slope can’t pass by reshaping sigmas.

  • rugnux: Fix de-novo rotation indexing committing a spurious axis-multiple supercell (collapsing to P1) via a cross-scheme smaller-cell tie-break on near-integer volume ratios.

  • rugnux: Widen refined-cell angle bounds to [30, 150] deg (rotation candidate and per-frame stills refinement); check refined angles against the reference cell.

  • rugnux: -S/--space-group now accepts a Hermann-Mauguin symbol (e.g. P43212) as well as a space-group number.

  • rugnux: Warn when the chosen cell/space group carries an indexing (merohedral) ambiguity needing a reference to resolve.

  • Indexing: Requesting the FFTW (CPU) indexer on a GPU node now fails with a clear, actionable message (rotation always uses the GPU FFT indexer there).

  • rugnux: Stills --refine-geometry[=N|off] - first-pass bundle-adjust of beam/distance/cell then re-index (default ON with a reference cell); accepts reference F/FP columns.

  • rugnux: Per-image geometry refinement -r flex tries all three algorithms per image and keeps the best (old name multi kept as an alias).

  • rugnux: Experimental stills partiality --still-partiality (Gaussian excitation-error) and --partiality-uncertainty <num> down-weighting the least-complete partials.

  • rugnux: Default the stills Bragg-integration box to r=6 (integration radii 6, 8, 12).

  • rugnux: Self-referenced stills scale in a single pass (fixing a weak-data collapse); a reference MTZ (-z) fixes SG/cell/ambiguity but never anchors the scale (stills and rotation).

  • rugnux: Cap normalised intensity (E^2) on second-lattice overlaps in the de-novo space-group search, so strong overlaps don’t skew the symmetry decision.

  • rugnux: Fix --scale on a self-contained _process.h5 (stored reflections and error model reload correctly).

  • rugnux: Add --spot-low-resolution <num> (default 50 A) and --min-pix-per-spot <num> (default 2) to tune spot finding on weak serial data.

  • jfjoch_viewer: Expose stills processing settings in the settings dock, rename geometry-refinement multi to flex, and refit the initial image on resize.

  • jfjoch_writer: Remove the CBF and TIFF image writers - only NXmx HDF5 is written (all three layouts remain).

  • Reader: Treat a negative total_flux in a stored dataset as unknown/absent rather than a valid flux.

  • Packaging: Build the self-contained Linux viewer against a static libdbus with glib disabled; add parallel image-build and in-container viewer-verification scripts.

  • rugnux: Write anomalous data as a standard CCP4 anomalous MTZ (one row per reflection: IMEAN, I(+)/I(-), F/F(+)/F(-)), readable by aimless/mtz2sca/ANODE.

  • rugnux: Always write merged reflections as both <prefix>.mtz and <prefix>.cif; the --scaling-output selector and text .hkl output are removed.

  • rugnux: Add a detector-plane modulation (flat-field) correction surface to rotation scaling (cross-validated, on by default; --no-scaling-corrections disables all), dropping R-meas.

  • rugnux: Add optional stills detector-plane modulation (--stills-modulation, default off) - the same cross-validated surface for the on-the-fly stills path.

  • Bragg integration: Local background is now a symmetric trimmed mean of the ring (--background-trim <f>, default 0.10; monochromatic), improving <I/sigma> and resolution-edge CC1/2.

1.0.0-rc.159

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux: Add --model model.pdb - score the merged data against an atomic model and compute initial maps. It reports R-work/R-free (scaling the model to the observed amplitudes with an overall scale, an anisotropic B and a flat bulk solvent - the standard few-parameter model, so a batch of maps stays directly comparable) and writes 2Fo-Fc / Fo-Fc electron-density maps (CCP4) plus a map-coefficient MTZ. The structure itself is not refined; the model is only re-fractionalised into the data cell.

  • rugnux: The merged reflection output now carries French-Wilson amplitudes (|F| and its sigma) next to the intensities - MTZ F/SIGF, mmCIF _refln.F_meas_au, and the text HKL - computed with the correct centric/acentric Wilson prior and epsilon multiplicity, so a downstream program (e.g. phenix.refine) can refine against amplitudes. The intensity columns are unchanged.

  • rugnux: R-free test-set flags are now assigned deterministically and consistently across symmetry - a Bijvoet pair I(+)/I(-) is never split between the work and free sets, and the assignment is a reproducible per-hkl hash that depends only on the reflection index, so every dataset of one crystal form gets the same ~5% free set (what a multi-dataset campaign such as PanDDA needs). On small data the fraction is floored so the test set stays large enough for a stable R-free (~500 reflections, capped at 10%); it stays flat at 5% on ordinary data. When a reference MTZ carries a FreeR_flag column its test set is imported instead, letting a whole campaign inherit one shared free set.

  • rugnux: A reference MTZ (--reference-mtz) can now fix the space group and cell for rotation data too (previously rejected), without being used to scale - the rotation merge stays self-consistent. When the crystal has an indexing (merohedral) ambiguity - a lattice symmetry higher than its Laue symmetry, e.g. P3/P4/P6/C2 - the reference also resolves it: each candidate reindexing (identity plus the twin-law cosets of the metric symmetry) is scored by its intensity correlation against the reference and the data are re-merged in the best-correlating one. This is a metric-preserving relabelling of hkl (the cell is unchanged) and a no-op for a holohedral crystal (which has no twin laws).

  • rugnux: --model validation now aligns the data to the model before scoring - the observed reflections are reindexed into the model’s enantiomorph when the two differ only by hand (indistinguishable from merged intensities). A merohedral indexing ambiguity is resolved against the reference MTZ when one is given (so a whole campaign shares one indexing convention); only with a model and no reference does validation fall back to fitting each candidate reindexing and keeping the lowest R-free.

  • rugnux: De-novo symmetry - recover a genuine high-symmetry group whose data are imperfectly scaled. Such a merge’s within-orbit chi² lands just past the self-consistency bound (each real symmetry step adds a little systematic scatter), right where a merohedral twin also lands, so the chi² ratio alone cannot separate them. The candidate is now rescued when the extra intensity-proportional systematic error it invokes stays small relative to the confirmed subgroup - a genuine symmetry step gains multiplicity without inflating the merge error model’s b, whereas a twin forces non-equivalent reflections together and b balloons. Fixes a cubic case (I23 instead of I222) with no change to any other crystal in the test battery, including the twins that must stay in their lower symmetry.

  • Docs: Document the French-Wilson amplitude estimation, R-free flagging, reference-based space-group/ambiguity resolution, and model-based validation/maps in CPU_DATA_ANALYSIS.md.

  • Frontend: The status-bar pill now shows a progress bar during detector calibration (previously only during measurement), and the calibration state and its button are labelled “Calibration”/“CALIBRATE” (the internal Pedestal state name is unchanged for back-compatibility).

1.0.0-rc.158

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (cos^3 of that angle) instead of cos^3(2*theta), so it is correct for a tilted detector and matches PyFAI solidAngleArray and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (XtalOptimizer) no longer silently ignores an imported PONI rot3 (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured rot3 through the full PONI rotation.

  • jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting \\); write processing outputs next to the input file, with a Browse button and independent _process.h5 / merged .mtz/.cif toggles; and show the determined space group in the merge-statistics window.

  • jfjoch_viewer: Connect to a broker over https (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow.

  • jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source.

  • rugnux: Accept an absolute -o output prefix in offline processing.

  • rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged).

  • rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing.

  • rugnux: De-novo symmetry robustness - don’t over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don’t flag twinning in holohedral Laue classes where no twin law can exist.

  • jfjoch_writer: Fold the refined beam centre into the NXmx detector translation vector too (not only the informational beam_center fields), so a reprocessed _process.h5 has a self-consistent refined geometry.

  • Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames.

  • Packaging: The self-contained Linux viewer .tgz now bundles cuFFT, so it runs without a system CUDA toolkit (.deb/.rpm are unchanged, distro-managed).

  • Docs: Documentation updated to match the current analysis code and CLI.

1.0.0-rc.157

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux: Rebrand the offline data-processing subsystem as rugnux and consolidate all offline analysis into the single rugnux binary - jfjoch_process is now rugnux, the former jfjoch_azint is now rugnux --azint-only, and jfjoch_scale is now rugnux --scale (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (--no-merge disables them), replacing the previous opt-in -M, --scale-merge.

  • rugnux: CLI fixes - default -N to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require --wavelength > 0, and correct the reproduced command line and --scale reference-cell handling.

  • rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics.

  • rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection’s rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below <I/sigma> = 1.

  • rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below --min-captured-fraction (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned).

  • rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved jfjoch prefix.

  • jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation.

  • jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a “reduce frame rate” warning.

  • jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint.

  • jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed).

  • jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types.

  • jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails.

  • jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak.

  • Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation.

  • CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*.

1.0.0-rc.156

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output.

  • jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed.

  • jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections.

  • jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw).

  • jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path.

  • jfjoch_process: Unify still and rotation processing on a single –force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental –reciprocal-profile option is removed.

  • jfjoch_process: Add EXPERIMENTAL ice-ring detection (–detect-ice-rings) that excludes ice reflections from scaling.

  • jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend.

  • jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata.

  • jfjoch_writer: Don’t write empty grid-scan position arrays when the dataset has no images.

  • Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don’t fit rather than aborting), and add the jfjoch_recompress tool.

  • jfjoch_viewer: Report “Multiple lattices detected” and grey out “Analyze dataset” on a live connection.

  • jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons.

  • CI: Build Windows (CUDA and non-CUDA) installers.

  • CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz.

1.0.0-rc.155

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D)

  • jfjoch_viewer: Some graphical improvements.

  • jfjoch_viewer: Simplify und unify data analysis settings.

  • jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker “dies” in the middle of data acquisition.

1.0.0-rc.154

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability)

1.0.0-rc.153

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing

  • jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files

  • jfjoch_broker: Add ROI calculation in non-FPGA workflow

  • jfjoch_broker: Fixes to TCP image pusher

  • jfjoch_broker: Remove NUMA bindings

  • jfjoch_broker: Improvements to indexing

  • jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values

  • jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors

  • jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs

  • jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results

1.0.0-rc.152

  • jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5)

  • jfjoch_viewer: Adjust Q bounds for azimuthal integration

  • jfjoch_azint: Add tool to do quick azimuthal integration

1.0.0-rc.151

  • jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname

1.0.0-rc.150

  • jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin.

  • jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins.

1.0.0-rc.149

  • XDS plugin: Fix HDF5 mutex to run on multiple processors

1.0.0-rc.148

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Improve azimuthal integration (add <I^2> calculation)

  • jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated)

  • jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin

1.0.0-rc.147

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015.

  • jfjoch_viewer: Add reciprocal space viewer

  • jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset

  • jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress)

1.0.0-rc.146

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension)

  • jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset.

  • jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling.

  • jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file.

  • jfjoch_scale: Dedicated tool for rescaling/merging existing data.

  • jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel.

WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being.

1.0.0-rc.145

This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

  • Default HDF5 writing mode is with VDS, not soft-links - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.

  • XDS plugin: Improve performance of VDS reading.

  • jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.

  • jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

1.0.0-rc.144

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M)

1.0.0-rc.143

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment

1.0.0-rc.142

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8

  • Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent

1.0.0-rc.141

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times

  • frontend: Fix selection of FFTW in indexing settings

1.0.0-rc.140

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection

  • jfjoch_broker: Minor bug fixes for rare conditions

  • jfjoch_process: Significantly improve performance

1.0.0-rc.139

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on /start

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on ‘/start’

  • jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite worklow preparation time

  • jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received

  • jfjoch_broker: Use nanoseconds internally for frame/image/readout time

  • jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished)

  • jfojch_broker: Improve profiling of different data analysis steps

  • jfjoch_broker: Record integration reflection count

  • jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds)

  • jfjoch_broker: Fix bug where ‘/wait_till_done’ got deadlocked

  • jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets

Breaking changes: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time.

1.0.0-rc.138

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time

  • jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations

  • jfjoch_broker: Goniometer axis name is converted to lowercase

  • jfjoch_broker: Fix bug, where wrong HTTP error codes were returned

  • jfjoch_process: Improve sigma estimation during merging (K. Takaba)

  • jfjoch_process: Modify spot finding thresholds

1.0.0-rc.137

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Better track time for each operation in the processing stack

  • jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress)

  • jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress)

  • jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0)

1.0.0-rc.136

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress).

1.0.0-rc.135

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4)

  • jfjoch_viewer: Improve image render performance

1.0.0-rc.134

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long

  • jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata

  • XDS plugin: Add generation of Jungfraujoch plugin for XDS

  • CI: Add tests with XDS and DIALS (xia2.ssx)

1.0.0-rc.133

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Use httplib for HTTP server instead of Pistache

  • jfjoch_broker: Drop OpenSSL support

  • jfjoch_broker: Base work for multi-lattice support in the future

  • jfjoch_broker: Improve recording time of data analysis steps

  • jfjoch_writer: Save per-image information about data analysis timing

  • Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib)

1.0.0-rc.132

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • Documentation: Fix equation rendering

1.0.0-rc.131

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS)

  • jfjoch_viewer: Fix calibration (pedestal) images being open flipped

  • jfjoch_process: Add space group detection (EXPERIMENTAL)

1.0.0-rc.130

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Rotation indexer has two retries if failes

  • jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot)

  • jfjoch_broker: Integration calculates background mask based on R2 radius

  • jfjoch_process: HDF5 files are not saved by default

1.0.0-rc.129

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files)

  • jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction

  • jfjoch_broker: Image pusher statistics are accessible via the REST interface

  • jfjoch_writer: Supports TCP image socket and for these auto-forking option

1.0.0-rc.128

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Handle properly reuse of image buffer locations

  • jfjoch_broker: Fix bug in counting idle slots

  • jfjoch_broker: Force obtuse angle for monoclinic cells

  • jfjoch_process: Change scaling refinement tolerance

1.0.0-rc.127

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Default EIGER readout time is 20 microseconds

  • jfjoch_broker: Multiple improvements regarding performance

  • jfjoch_broker: Image buffer allows to track frames in preparation and sending

  • jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer

  • jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets

  • jfjoch_writer: Fixes regarding properly closing files in long data collections

  • jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code

1.0.0-rc.126

This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg.

1.0.0-rc.125

This is an UNSTABLE release. This version adds scalign and merging. These are experimental at the moment, and should not be used for production analysis. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Improve logic on switching on/off spot finding

  • jfjoch_broker: Increase maximum spot count for FFBIDX to 65536

  • jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD)

  • jfjoch_process: Add scalign and merging functionality - program is experimental at the moment and should not be used for production analysis

  • jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection

  • jfjoch_writer: Save more information about each reflection

1.0.0-rc.124

This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123.

  • jfjoch_broker: Improve refection position prediction and Bragg integration code.

  • jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation.

  • jfjoch_writer: Fix saving mosaicity properly in HDF5 file.

  • jfjoch_viewer: Introduce high-dynamic range mode for images

  • jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%)

  • jfjoch_viewer: Zoom-in numbers have better readability

1.0.0-rc.123

This is an UNSTABLE release.

  • jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility

  • jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Add more color schemes for better image readability

  • HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable

  • HDF5: suppress warning if path (upstream group) doesn’t exists when checking if leaf exists

1.0.0-rc.122

This is an UNSTABLE release.

  • jfjoch_broker: Add thresholding to prefer shorter vectors after FFT

  • jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress)

  • jfjoch_broker: Update nlohmann::json to 3.12.0

  • jfjoch_viewer: Display file opening errors

  • jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available

1.0.0-rc.121

This is an UNSTABLE release.

  • jfjoch_broker: Report changes in the image buffer, so viewer doesn’t reload constantly

  • jfjoch_viewer: Improve performance of loading images

  • jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes

  • jfjoch_viewer: Auto-foreground calculated with histogram

  • jfjoch_viewer: Fix rare segmentation fault

1.0.0-rc.120

This is an UNSTABLE release.

  • jfjoch_broker: Improve performance of binary plot export

1.0.0-rc.119

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Add binary export of data analysis plots over OpenAPI

  • jfjoch_broker: Minor fixes to HTTP error handling

  • jfjoch_viewer: Prefer binary plots over JSON plots

  • jfjoch_viewer: Change foreground with F button + wheel

  • jfjoch_viewer: Change way how angles are displayed

  • jfjoch_viewer: Display resolution of the mouse cursor in top left corner

1.0.0-rc.118

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement.

  • jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations

1.0.0-rc.117

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add ROI results to the dataset info plots

  • jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment

1.0.0-rc.116

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add binning options in the context menu

1.0.0-rc.115

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Default spot finding settings can be configured via config JSON

  • jfjoch_viewer: FFT analysis of data in the dataset plot

1.0.0-rc.114

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Fix generating JPEG images with resolution estimation

1.0.0-rc.113

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Improve handling of rotation indexing

  • jfjoch_broker: More information saved in CBOR end message (WIP)

  • jfjoch_writer: Save rotation indexing lattice parameters and Niggli class

  • jfjoch_viewer: Remove (for now) primitive cell information

  • jfjoch_viewer: Use angle for dataset info plot for rotation scans

1.0.0-rc.112

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Experimental rotation (3D) indexing

  • jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values

1.0.0-rc.111

This is an UNSTABLE release.

  • jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment)

  • jfjoch_viewer: Fix auto contrast not refreshing image

1.0.0-rc.110

This is an UNSTABLE release.

  • jfjoch_broker: Add auto-contrast option for preview images

  • Frontend: Add logo image

  • jfjoch_viewer: Add logo image

  • jfjoch_viewer: For image chart allow to set min value to zero

  • jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure

  • jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there)

  • Documentation: Add logo image

1.0.0-rc.109

This is an UNSTABLE release.

  • jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard

  • jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors

1.0.0-rc.108

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN)

  • jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default

  • jfjoch_viewer: Fit to window of image is now default when size of the grid changes

1.0.0-rc.107

This is an UNSTABLE release.

  • jfjoch_viewer: Minor polishing of new functionality

  • jfjoch_broker: User NaN for empty azimuthal bins

1.0.0-rc.106

This is an UNSTABLE release.

  • jfjoch_viewer: Allow for multiple dataset info plots

  • jfjoch_viewer: Highlight current element in grid

1.0.0-rc.105

This is an UNSTABLE release.

  • jfjoch_viewer: Clean-up widgets slightly

  • jfjoch_viewer: Limit right panel to 600 pixels

  • jfjoch_viewer: Parse crystal symmetry type

  • jfjoch_viewer: Grid scan view takes color map and can be fit to zoom

1.0.0-rc.104

This is an UNSTABLE release.

  • jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier)

  • jfjoch_viewer: Display grid scan results in 2D (work in progress)

  • jfjoch_viewer: Improve auto-scaling on start of images (work in progress)

  • jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots

1.0.0-rc.103

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

  • jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates

1.0.0-rc.102

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

1.0.0-rc.101

This is an UNSTABLE release.

  • jfjoch_viewer: Auto load is better handling change of states

  • jfjoch_viewer: Fix DBus registration

  • jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update

  • jfjoch_viewer: Calculate ROI in a more efficient way

1.0.0-rc.100

This is an UNSTABLE release.

  • jfjoch_viewer: Fix dbus registration

  • jfjoch_viewer: Remove background slider for diffraction image

  • jfjoch_viewer: Adjustments for 2D azimuthal image viewer

1.0.0-rc.99

This is an UNSTABLE release.

  • jfjoch_broker: Fix output during mask data collection

1.0.0-rc.98

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization

1.0.0-rc.97

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask

  • jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems)

  • jfjoch_viewer: Add option to handle user pixel mask (experimental)

  • jfjoch_viewer: More options for ROI

  • jfjoch_viewer: Add window to display calibration

1.0.0-rc.96

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • jfjoch_broker: Remove PNG preview, no dependency on libpng

  • jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds)

  • jfjoch_viewer: Show data collection time in dataset tooltip

  • jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate)

  • jfjoch_viewer: Auto foreground value

  • Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds

  • Remove OpenBLAS dependency

1.0.0-rc.95

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • Add git-lfs to Rocky8 docker image

Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved.

1.0.0-rc.94

This is an UNSTABLE release.

  • FFTIndexer: Add limit on angles to avoid colinear vectors

  • Docker images: Add 3D Qt

  • Gitea: Fixes to the pipeline

1.0.0-rc.93

This is an UNSTABLE release.

  • CI: Fixes to Gitlab based pipeline

  • PCIe driver: Fix PCIe revision being hex number

1.0.0-rc.92

This is an UNSTABLE release.

  • jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample.

1.0.0-rc.91

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated

  • jfjoch_broker: Spot finding and integration predictions are ported to a GPU

  • jfjoch_broker: Estimate resolution

  • jfjoch_broker: Lattice search

  • jfjoch_broker: Many more improvements in image analysis

1.0.0-rc.90

This is an UNSTABLE release.

  • jfjoch_broker: for indexing min index spots for a viable cells can be changed via OpenAPI

  • jfjoch_viewer: Optional auto-reanalyze images

  • jfjoch_writer: Add option where no files at all are saved

  • Documentation: improvements

1.0.0-rc.89

This is an UNSTABLE release.

  • jfjoch_broker: Fix resolution estimation code

  • jfjoch_broker: Fix Wilson B-factor calculation code

  • jfjoch_viewer: Improve display of plots

  • jfjoch_viewer: Fix segmentation fault

  • jfjoch_viewer: Display missing metadata when using HTTP

  • jfjoch_viewer: Fix bug when opening the same file twice

1.0.0-rc.88

This is an UNSTABLE release.

  • jfjoch_viewer: Add resolution estimation to the image information

  • jfjoch_broker: Minor changes to resolution estimate routine

1.0.0-rc.87

This is an UNSTABLE release.

  • jfjoch_viewer: Display more image metadata (angle / exposure time)

  • jfjoch_viewer: Improve I/sigma and B-factor plots

  • jfjoch_broker: Estimate resolution based on visible spots

1.0.0-rc.86

This is an UNSTABLE release.

  • jfjoch_broker: Update logic when initializing detector to make it a bit more resilient

  • Gitea pipelines have nocuda option for all architectures

1.0.0-rc.85

This is an UNSTABLE release.

  • jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image

  • jfjoch_viewer: Handle spot count better in dataset info plots

  • jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled

1.0.0-rc.84

This is an UNSTABLE release.

  • jfjoch_broker: Write in log which detector is being initialized

  • Changes to automated build system

1.0.0-rc.83

This is an UNSTABLE release.

  • jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before)

  • CI: Fix script to generate python client

1.0.0-rc.82

This is an UNSTABLE release.

  • jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment)

  • Frontend: Minor fixes

  • Build scripts: Minor fixes to FFTW

1.0.0-rc.81

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing

  • jfjoch_broker: Add FFTW based indexer for CPU only indexing

  • jfjoch_broker: Enable saving X-ray fluorescence spectra

  • jfjoch_writer: Write total spot count (before filtering)

  • jfjoch_viewer: Add more information on source, sample, and buttom to show ice rings

  • jfjoch_viewer: Enable data processing inside the viewer

CI: Moving from Gitlab to Gitea at PSI

1.0.0-rc.80

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer

1.0.0-rc.79

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance

  • jfjoch_writer: Save nimages_per_trigger in detectorSpecific

1.0.0-rc.78

This is an UNSTABLE release.

  • jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore)

1.0.0-rc.77

This is an UNSTABLE release.

  • jfjoch_viewer: Display detector and dataset settings with tooltips

  • jfjoch_viewer: Clean excessive HDF5 warnings

  • jfjoch_viewer: Display unit cell

  • jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset

1.0.0-rc.76

This is an UNSTABLE release.

  • jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones

  • jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median)

  • jfjoch_writer: Save space group number (non-NXmx addition) in addition to name

  • jfjoch_viewer: Fix the bug on reading space_group as string

  • jfjoch_viewer: Add missing resolution labels on rings

  • jfjoch_viewer: Remove Q value from the status bar

1.0.0-rc.75

This is an UNSTABLE release.

  • jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being

1.0.0-rc.74

This is an UNSTABLE release.

  • jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing)

  • jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization

1.0.0-rc.73

This is an UNSTABLE release.

  • jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default

1.0.0-rc.72

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Refactor of indexing and geometry refinement code

  • jfjoch_broker: Handle space group/centering in refinement code

  • jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images

  • jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion

  • jfjoch_writer: Experimental saving of reflections

  • jfjoch_writer: Save space group name as string

  • jfjoch_viewer: Add profile radius and B-factor

  • jfjoch_viewer: Show 4 digits for wavelength

  • jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings)

  • FPGA: Use UDP destination port to distinguish between detector modules and data streams

  • FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up)

  • FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene)

1.0.0-rc.71

This is an UNSTABLE release.

  • jfjoch_broker: Remove resolution estimation via machine learning

  • jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers

  • jfjoch_viewer: Fix error when HDF5 files with indexing results couldn’t be loaded on a machine without GPU

1.0.0-rc.70

This is an UNSTABLE release. This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement

  • jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards)

  • jfjoch_broker: Enable geometry refinement - first try (work in progress)

  • jfjoch_viewer: Fix deadlock when opening HTTP connections

  • jfjoch_viewer: Display rings as ellipses with detector tilt

  • jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image

  • jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3)

  • Add Google Ceres a non-linear least-square optimization library to Jungfraujoch

  • Add experimental detector calibration routines (for LaB6)

  • Improve documentation on the ZeroMQ writer notification socket and detector geometry

1.0.0-rc.69

This is an UNSTABLE release.

  • jfjoch_viewer: Metadata can be modified for an open dataset (no option to save)

  • jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments

  • jfjoch_viewer: Resolution rings not enabled by default

  • jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file

1.0.0-rc.68

This is an UNSTABLE release.

  • jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector

  • jfjoch_broker: Default detector settings can be configured for each detector separately

  • jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings

  • jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing

  • jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter

  • jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI

1.0.0-rc.67

This is an UNSTABLE release.

  • jfjoch_broker: Enable SSL

  • jfjoch_broker: Wilson B-factor only provided is fit is relatively OK (R^2 > 0.3); this will be refined much more in the future

1.0.0-rc.66

This is an UNSTABLE release.

  • jfjoch_broker: Indexers operate as thread pool, which is operating

  • jfjoch_viewer: Increase interval between loading images + fix too many verbose messages

1.0.0-rc.65

This is an UNSTABLE release.

  • jfjoch_broker: Print information regarding used image pushers

  • jfjoch_viewer: Allow syncing with Jungfraujoch server

  • OpenAPI: Clarify licensing terms in the file

1.0.0-rc.64

This is an UNSTABLE release.

  • jfjoch_broker: Fix issue in receiver light with very long preparation time for threads

  • jfjoch_broker: Add verbose option

  • jfjoch_broker: Don’t trigger pedestal if critical settings are not changed when loading detector settings

  • jfjoch_broker: Detector left in busy state when detector settings were improper

  • jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times

  • jfjoch_lite_perf_test: Add verbose option

1.0.0-rc.63

This is an UNSTABLE release.

  • jfjoch_broker: Save NX/NY for grid scan result

  • jfjoch_broker: Add processing time to CBOR output and plot

  • jfjoch_writer: Add processing time to data file

1.0.0-rc.62

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where low resolution spots were not counted properly

  • jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count

  • jfjoch_broker: Add more spot count information to CBOR

  • jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles

1.0.0-rc.61

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector

  • jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results

  • jfjoch_broker: High resolution limit for low res. spot counting can be adjusted

1.0.0-rc.60

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released

  • jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices

  • jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength

1.0.0-rc.59

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock.

  • jfjoch_writer: Verbose option for debugging.

1.0.0-rc.58

This is an UNSTABLE release.

  • jfjoch_viewer: Fix memory leak

  • jfjoch_writer: Add detector_number/serial_number to master file

1.0.0-rc.57

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible

  • jfjoch_viewer: “Movie” mode

1.0.0-rc.56

This is an UNSTABLE release.

  • jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation

1.0.0-rc.55

This is an UNSTABLE release.

  • jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation

1.0.0-rc.54

This is an UNSTABLE release.

  • jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not)

  • jfjoch_broker: Don’t remove shared memory area when deactivating detector

  • jfjoch_writer: Save writer release

  • jfjoch_viewer: Increase time for the messages in the status bar

1.0.0-rc.53

This is an UNSTABLE release.

  • PCIe driver: Imperfect solution for RHEL 9.5+ changes

  • jfjoch_writer: Fix to angle containers for AutoProc compatibility

  • jfjoch_fpga_test: Use consecutive number for devices, not interleaved

1.0.0-rc.52

This is an UNSTABLE release.

  • jfjoch_viewer: Use warmer colors to distinguish from AareGUI

  • jfjoch_viewer: Minor adjustments to DBus setting image number

  • jfjoch_broker: Fix in low resolution spot count plotting

1.0.0-rc.51

This is an UNSTABLE release.

  • jfjoch_broker: Send preview in PNG format

  • jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range

  • jfjoch_broker: Provide ML resolution estimation in scan result

  • jfjoch_broker: Allow removing beam center in web preview

1.0.0-rc.50

This is an UNSTABLE release.

  • The release fixes some of many bugs introduced in recent releases

  • jfjoch_viewer: display predictions for indexed cells

1.0.0-rc.49

This is an UNSTABLE release.

  • jfjoch_broker: Handle sample temperature (K) and ring current (mA) to metadata

  • jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current

1.0.0-rc.48

This is an UNSTABLE release.

  • jfjoch_broker: fix the bug when a unit cell was not exported for a scan result.

1.0.0-rc.47

This is an UNSTABLE release.

  • jfjoch_viewer: fix dbus service path

  • jfjoch_writer: fix CBF/TIFF writing

1.0.0-rc.46

This is an UNSTABLE release.

  • jfjoch_viewer: remove dependency on image analysis

1.0.0-rc.45

This is an UNSTABLE release.

  • jfjoch_broker: Detector list returns pixel size (mm)

1.0.0-rc.44

This is an UNSTABLE release.

  • jfjoch_broker: more general definition of scan result export

Braking changes:

  • It removes additions to OpenAPI from 1.0.0-rc.43

  • It makes changes to the “unit_cell” definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations.

1.0.0-rc.43

This is an UNSTABLE release.

  • jfjoch_broker: Export grid scan results into a single data structure

1.0.0-rc.42

This is an UNSTABLE release.

  • jfjoch_broker: Add pixel_sum to CBOR output.

  • jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine

  • jfjoch_writer: Save pixel_sum

1.0.0-rc.41

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction

  • jfjoch_broker: Export GPU count via OpenAPI

  • jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector

  • FPGA: Increase the number of integration bins to 2048

1.0.0-rc.40

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures

  • jfjoch_broker: Improve metadata for rotation data collection

  • jfjoch_broker: Better handling of plotting

  • jfjoch_broker: FFT based indexing

  • jfjoch_broker: Integration, first try, results not saved at the moment

  • jfjoch_broker: Internal improvements in image handling

  • jfjoch_writer: Multiple adjustments adapt to changes in this release for new features

  • jfjoch_writer: New state management model to improve clarity of error reporting

  • jfjoch_viewer: Remote control via DBus

  • Frontend: Multiple adjustments for new features

  • Frontend: Grid scan plots

WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken.

1.0.0-rc.39

  • FPGA: Bugfix for pixel masked for data analysis if summation was on

  • jfjoch_viewer: Fix segmentation fault when cursor was outside of image

1.0.0-rc.38

  • jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via RES

  • jfjoch_broker: Neural net model can use all 4 quadrants of the detector

  • jfjoch_broker: For EIGER image time can be provided through /start

  • jfjoch_viewer: Add image list option

  • jfjoch_viewer: Drawing circular ROIs with shift

  • jfjoch_viewer: Enable image summation

  • jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer

1.0.0-rc.37

  • jfjoch_broker: Make locking rules more flexible

  • jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors

  • jfjoch_viewer: Add status bar

1.0.0-rc.36

This is UNSTABLE release. Wait for new version to use in a production environment.

  • jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code)

  • jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation)

  • jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data

  • jfjoch_writer: Increase buffering to 1000 images in the receiver

  • jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5

1.0.0-rc.35

This is UNSTABLE release, not properly tested. Wait for new version for using production.

  • jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting.

  • jfjoch_writer: Save EIGER energy threshold

  • jfjoch_writer: Add /entry/sample/goniometer for compatibility with eiger2cbf program

1.0.0-rc.34

This is UNSTABLE release - introducing new features, but not properly tested. Wait for new version for using production.

  • jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings)

  • jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells

  • jfjoch_broker: Handle situation when detector doesn’t want to gracefully stop (to be tested)

  • jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots

  • jfjoch_viewer: Can extract data analysis results from “legacy” format

  • jfjoch_viewer: Display dataset name

  • FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels

  • FPGA: Add pixel sum to module statistics

  • FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI

  • FPGA: Spot finder is back to full dynamic range (24-bit)

  • FPGA: More debug features for internal FIFOs

Known issues:

  • ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts.

  • EIGER data acquisition has an issue that is currently debugged

1.0.0-rc.33

  • jfjoch_broker: Fix issue with EIGER settings being loaded improperly

1.0.0-rc.32

  • jfjoch_broker: Refactor code for azimuthal integration for further improvements

  • jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later)

  • jfjoch_writer: Fix too much verbose information

  • FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20)

1.0.0-rc.31

This is UNSTABLE release - introducing many features, but still needs more testing. Expecting soon to put bugfix release.

  • jfjoch_writer: Allow to enable overwriting existing files (not enabled by default)

  • jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default)

  • jfjoch_viewer: Image viewer work early test version

  • jfjoch_broker: Fixes to counting packets per dataset/image

  • jfjoch_broker: Image buffer is accessible for outside to check images

  • jfjoch_broker: error/saturated pixels and dedicated ROI “beam” can be tracked online

  • jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU

  • jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation

  • jfjoch_broker: Fix bug in EIGER initializing

  • jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot

  • PCIe driver: Add PCIe link speed and width

  • FPGA: Improve counting error/saturated/min/max pixels

  • FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas

  • FPGA: Require Vivado 2022.2

Warning: There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency.

1.0.0-rc.30

  • jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to “loose” images within the writer

1.0.0-rc.29

  • jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU

  • jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file

  • jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info)

  • jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly

Warning - two breaking changes to OpenAPI:

  • Handling of ROIs is through /config/roi path only for both circle and box ROIs, path in /roi are no longer accessible

  • broker_status structure introduced in 1.0.0-rc.28 has member message and not error_message to allow handling info/warning messages as well

1.0.0-rc.28

  • jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI

  • jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not complaint to OpenAPI specs

  • jfjoch_test: remove header that failed when CUDA is absent during compilation

  • frontend: add soft trigger button in data collection tab

  • frontend: show error message when in error state

  • CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF)

1.0.0-rc.27

  • jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation

  • jfjoch_broker: update pistache web server version

  • jfjoch_writer: minor changes to republish logic

  • Improvements to documentation

1.0.0-rc.26

  • jfjoch_broker: implement ZeroMQ stream for image metadata information

  • jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent

  • jfjoch_broker: add crystal lattice plots

  • jfjoch_broker: remove empty bins from the plots

  • jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU “long” summation

  • jfjoch_broker: Fix bug when mean background estimation / indexing rate where affected by previous experiment

  • jfjoch_writer: fix missing “-w” parameter

  • jfjoch_writer: temporary files have “.tmp” suffix

  • jfjoch_writer: refactor logic for watermarks

  • jfjoch_writer: report on internal FIFO utilization

  • jfjoch_writer: clean-up naming for azimuthal integration and background estimate

  • jfjoch_writer: write final background estimate and indexing rate in the master file

  • tools/: remove unnecessary tools, make naming consistent

  • CBOR: Add indexing rate and background estimate to end message

  • CBOR: Clean-up documentation

1.0.0-rc.25

  • Updates to documentation

  • License set to GPLv3 / OHL-S

  • Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms

  • Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5

1.0.0-rc.24

New FPGA functionality:

  • EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit “real” depth is 23-bit + 1-bit signed)

  • Output possible to 8, 16 and 32-bit data

  • Threshold is applied before summation

  • Pixel mask can be applied on FPGA

  • Mark pixels with ADC content = 0 as bad pixels

  • FPGA stores semantic version information (access via /sys/class/misc/jfjoch…/version)

New software functionality:

  • Long summation (above 256 frames) done on CPU

  • Mechanism to save arbitrary data to HDF5 file

  • ZeroMQ preview has option to send start message

  • Rework pixel mask + add statistics displayed in web interface

Bug fixes:

  • Web frontend: Update preview image automatically during data acquisition

  • jfjoch_broker: Error handling if CUDA driver is not installed

  • jfjoch_broker: Correctly update progress during pedestal

  • jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF

  • jfjoch_action_test: enable HLS simulation

Documentation improvement and placement in a dedicated directory

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS.html b/CPU_DATA_ANALYSIS.html new file mode 100644 index 00000000..c596961d --- /dev/null +++ b/CPU_DATA_ANALYSIS.html @@ -0,0 +1 @@ + CPU-side crystallographic data analysis (Jungfraujoch) — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

CPU-side crystallographic data analysis (Jungfraujoch)

This document describes the crystallographic algorithms implemented in Jungfraujoch for CPU- and GPU-side real‑time and near‑real‑time data analysis.

Scope. The pipeline covered here comprises:

  1. geometry mapping and corrections,

  2. azimuthal integration (powder/radial profiles),

  3. Bragg spot finding (strong pixels → connected components → spot descriptors),

  4. indexing (still and rotation modes),

  5. Bravais lattice / centering inference,

  6. geometry and lattice refinement,

  7. reflection prediction (still and rotation),

  8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free),

  9. scaling and merging,

  10. merge-level error modelling, outlier rejection and the resolution cutoff,

  11. space-group determination from the merged intensities (Laue group, screw axes, centering) and the twinning check,

  12. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref),

  13. amplitude estimation (French–Wilson) and R-free test-set flagging,

  14. optional model-based validation: R-free against a supplied model and 2Fo−Fc / Fo−Fc electron-density maps.

References

The methods draw on, and in places reimplement, solutions from:

  • W. Kabsch, “XDS”, Acta Cryst. D66 (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts).

  • W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, Acta Cryst. D66 (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors).

  • T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts).

  • J. Kieffer & J. P. Wright, “PyFAI: a Python library for high performance azimuthal integration on GPU”, Powder Diffraction 28 (2013), S339-S350 (detector geometry definition, azimuthal integration)

  • H. Powell, “The Rossmann Fourier autoindexing algorithm in MOSFLM”, Acta Cryst. D55 (1999), 1690-1695 (FFT indexing)

  • S. French & K. Wilson, “On the treatment of negative intensity observations”, Acta Cryst. A34 (1978), 517-525 (Bayesian amplitude estimation from intensities).

  • A. T. Brünger, “Free R value: a novel statistical quantity for assessing the accuracy of crystal structures”, Nature 355 (1992), 472-475 (R-free cross-validation).

  • M. Wojdyr, “GEMMI: A library for structural biology”, J. Open Source Softw. 7 (2022), 4200 (model / structure-factor / map machinery used in §14).

  • J. P. Wright, “Experiences with GPU decompression for bitshuffle + LZ4 data”, HDF5 User Group meeting (2021), and github.com/jonwright/bslz4decoders (device-side decoding of bitshuffle+LZ4 images, §0).

  • Z. Otwinowski & W. Minor, “Processing of X-ray diffraction data collected in oscillation mode”, Methods Enzymol. 276 (1997), 307-326 (reweighted, de-biased profile-fit variances).

  • G. Winter et al., “DIALS: implementation and evaluation of a new integration package”, Acta Cryst. D74 (2018), 85-97, and J. Beilsten-Edmands et al., Acta Cryst. D76 (2020), 385-399 (CC1/2 resolution cutoff, merge outlier rejection, scaling error model).

  • P. Evans, “Scaling and assessment of data quality”, Acta Cryst. D62 (2006), 72-82, and P. R. Evans, Acta Cryst. D67 (2011), 282-292 (POINTLESS: operator-by-operator point-group scoring, and the axial-zone screw-absence test).

  • A. G. W. Leslie & H. R. Powell, “Processing diffraction data with MOSFLM” (2007), NATO Science Series II 245, 41-51 (post-refinement practice: what is refined per image and what over a wedge).

  • K. Diederichs & P. A. Karplus, Nat. Struct. Biol. 4 (1997), 269-275, and P. A. Karplus & K. Diederichs, Science 336 (2012), 1030-1033 (R_meas / R_pim, CC1/2 and CC*).

  • IUCr Commission on Crystallographic Nomenclature, “Statistical descriptors in crystallography”, Acta Cryst. A45 (1989), 63-75, and Acta Cryst. A51 (1995), 565-569 (uncertainty conventions).

(list is not exhaustive; the full citations, with DOIs, are in ACKNOWLEDGEMENT.md)

0. Getting the image onto the GPU: device-side bitshuffle+LZ4 decoding

Images arrive bitshuffle+LZ4 compressed (HDF5 filter 32008), and everything from §1 onwards runs on the GPU when one is present. Instead of decompressing on the host and uploading the image, the compressed chunk is uploaded — a few MB rather than tens of MB — and decoded on the device. The approach follows Jon Wright (ESRF); the kernels are Jungfraujoch’s own.

Two kernels do the work:

  1. LZ4, one warp per bitshuffle block. Blocks are independent, so the parallelism is across them; within a warp every lane runs the same sequence parser over the same bytes, and the literal and match copies are split across the 32 lanes so the stores coalesce. An overlapping match is treated as a pattern of period offset sourced from bytes that already precede the write position, which keeps it parallel rather than a serial byte loop; offset == 1 (a run of one repeated byte, the common case in sparse detector data) and power-of-two offsets avoid the modulo altogether. Because the lanes cooperate on the copies, each one is followed by __syncwarp() — a later match can read bytes another lane wrote, and since Volta that ordering is not implicit.

  2. The bitshuffle inverse fused with preprocessing. One thread owns one group of 8 elements across every byte-plane, so once it has transposed its 8 bytes out of each plane it holds 8 complete elements — and it applies the pixel mask, the error marker and the saturation cap and emits 8 finished int32 pixels directly. The decompressed image is therefore never materialised in device memory at all, which removes a frame-sized buffer per worker and a full-frame write plus read from the pipeline. Staging nothing in shared memory also means the kernel has no dynamic-shared-memory request, so it is indifferent to the bitshuffle block size the file declares. For 8-bit images there is a single plane and the assembly degenerates to a copy.

The block offsets inside the container can only be discovered by reading the block lengths in order, so that scan stays on the host.

Only BSHUF_LZ4 is decoded on the device. For the zstd variants (BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF), and for uncompressed or float images, BSLZ4DecoderGPU::Supports() returns false and the pipeline decompresses on the host and uploads as before.

The container arrives off the network or off disk and is not trusted. Everything checkable on the host — declared sizes, the block scan, a block size that is not a multiple of 8 elements, a block count the chunk could not hold, trailing bytes — is rejected before any work is queued; the kernel additionally flags a block that did not decode to exactly its declared length, which becomes an exception once the caller has synchronised. That last check matters because the decode buffers are reused frame to frame: a block that stopped early would leave the previous image’s most significant byte-plane in place, which reads not as a missing corner but as real pixels several powers of two too bright.

1. Geometry, reciprocal-space mapping, and basic quantities

1.1 Coordinate conventions

For a pixel coordinate \((x,y)\) (in pixels), Jungfraujoch converts to a laboratory direction vector via:

  1. shift by direct-beam position \((x_\mathrm{beam}, y_\mathrm{beam})\),

  2. scale by pixel size \(p\) (mm),

  3. set detector distance \(D\) (mm),

  4. apply detector orientation rotation \(R_\mathrm{det}\) (PyFAI-like parameterization).

The unnormalized detector coordinate (mm) is: \( \mathbf{r}_\mathrm{det}(x,y) = \begin{pmatrix} (x-x_\mathrm{beam})p\\ (y-y_\mathrm{beam})p\\ D \end{pmatrix}. \)

The lab-frame vector is: \( \mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. \)

Let the incident wavevector magnitude be \(k = 1/\lambda\) in Å\(^{-1}\), and define: \( \mathbf{S}_0 = (0,0,k). \)

The reciprocal-space scattering vector associated with pixel \((x,y)\) is: \( \mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. \)

This \(\mathbf{s}\) is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement.

1.2 Two-theta, azimuth, resolution and \(q\)

The scattering angle \(2\theta\) is computed from \(\mathbf{r}_\mathrm{lab}\) via: \( 2\theta = \arctan\!\left(\frac{\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2}}{z_\mathrm{lab}}\right). \)

Resolution (Å) at a pixel is: \( d = \frac{\lambda}{2\sin\theta}. \)

The magnitude \(q = 2\pi/d\) is used for radial binning and ice-ring handling.

1.3 Distance from the Ewald sphere

For a reciprocal lattice point \(\mathbf{p}\) (Å\(^{-1}\)), define: \( \Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. \) Jungfraujoch uses \(|\Delta_\mathrm{Ewald}|\) as an operational proxy for excitation error. This appears in:

  • still prediction (accept if \(|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}\)),

  • profile radius estimation (see §11.1),

  • still partiality option in scaling/merging (§10.2).

1.4 Measuring the direct beam before indexing

The beam centre in the file is often a placeholder, and nothing else measures it until post-refinement (§7.5) — by which time a wrong centre has already chosen the lattice. With --estimate-beam-center it is measured from spot positions alone, before anything is indexed. Two exact facts about a rotation sweep supply the two coordinates:

Friedel mates half a turn apart. The Laue condition fixes the component of \(\mathbf{q}\) along the beam, \(q_\parallel = -\lVert\mathbf{q}\rVert^2\lambda/2\). Rotating 180° about the spindle \(\mathbf{m}\) negates the two components perpendicular to \(\mathbf{m}\) and taking \(-h\) negates all three, so together they negate only the component along \(\mathbf{m}\) and leave \(q_\parallel\) untouched. With the spindle perpendicular to the beam, \(-h\) therefore diffracts at \(\varphi+180°\) exactly where \(h\) diffracts at \(\varphi\), and its spot sits at the mirror image of \(h\)’s along the spindle. This gives the beam coordinate along the spindle. Only the reciprocal lattice’s centrosymmetry is needed for the geometry; Friedel’s law \(|F(h)|=|F(-h)|\) is used separately, to tell a true pairing from an accidental one.

The second crossing. The same reflection meets the Ewald sphere twice, at two angles that are generally not 180° apart, differing only in the sign of the lab component perpendicular to both \(\mathbf{m}\) and the beam. This gives the remaining coordinate. The two crossings are separated by a sweep angle fixed by the reflection’s own position, which is what identifies genuine pairs.

Neither observable requires a cell or an orientation matrix: each candidate pairing votes for a beam coordinate, and the true value accumulates while wrong pairings scatter. A Friedel pair needs both \(\varphi\) and \(\varphi+180°\) recorded, so a sweep of \(S°\) yields only \(S-180\) degrees’ worth of pairs — a sweep of exactly half a turn yields none, and the estimator is refused below a floor on that span.

The mirror is exact in the laboratory frame, so it is sensitive to the spindle direction. A skew of the spindle about the beam spreads the vote rather than shifting it, and is fitted alongside the centre (--no-fit-spindle keeps the axis from the file); a tilt of the spindle towards the beam is measured and reported but not applied, being confounded with the detector tilt until that is fitted too. Nothing inside the fit can tell that the vote settled on the wrong periodic maximum — every frame pair agrees with every other — so the answer is accepted only if it does not move when the search is started from a different position. Frames are sampled away from both ends of the sweep, where shutter synchronisation can spoil an image.

Where the sweep is shorter than half a turn the spot symmetry cannot be formed, and the centre is taken instead from the centroid of the radial background profile, which needs only a few images. Where neither method can measure the centre, the value from the file is kept.

2. Azimuthal integration (radial profiles)

Azimuthal integration produces a radial profile \(I(q)\) or \(I(d)\) by histogramming pixels into radial bins. Pixels are not split across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 \(\phi\) sectors (azim_bins, --azim-phi-bins), giving a 2D \(q\times\phi\) profile that exposes azimuthal anisotropy such as detector shadowing or sample texture.

2.1 Histogram estimator

Let bin index \(b(x,y)\) be precomputed from \(q(x,y)\) (or equivalently from \(d(x,y)\)) and, when \(\phi\) sectors are enabled, the azimuth \(\phi(x,y)\) — so \(b = b_q + b_\phi B_q\). For each bin \(b\):

  • accumulate corrected intensity and its square: \( S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, \)

  • and count: \( N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. \)

The profile reports both the mean \(\bar{I}_b = S_b / N_b\) (when \(N_b>0\)) and a per-bin sample standard deviation \(\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}\) (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded.

2.2 Corrections applied

Two standard corrections are available:

(i) Solid angle / geometric correction. A flat pixel’s solid angle falls off with the incidence angle \(\alpha\) between the scattered ray and the detector normal. With the in-plane detector offsets \(u=(x-x_\mathrm{beam})p\) and \(v=(y-y_\mathrm{beam})p\) (§1.1) and detector distance \(D\), \( \cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad C_\Omega = \cos^3\alpha, \) applied — like the polarization term below — as a divisor (intensities are scaled by \(1/\cos^3\alpha\)), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because \(\alpha\) is evaluated in the detector’s own frame it is invariant under detector tilt (\(\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}\)), matching PyFAI’s solidAngleArray and MAX IV azint. It reduces to the commonly quoted \(\cos^3(2\theta)\) form only for an untilted detector, where the incidence angle coincides with the scattering angle.

(ii) Polarization correction. With polarization coefficient \(P\) (beamline dependent) and azimuth \(\phi\): \( C_\mathrm{pol}(2\theta,\phi) = \frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), \) applied as a divisor to intensities (i.e. scale by \(1/C_\mathrm{pol}\)) when enabled.

2.3 Background estimate for profiles

A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid \(Q\) window (default \(2\pi/5\) to \(2\pi/3\) Å\(^{-1}\)). This background is used for monitoring and diagnostics; it is not the same as the local Bragg-spot background used in summation integration (§9.2).

3. Spot finding (strong pixels → Bragg spots)

Spot finding is a two-stage process:

  1. Strong-pixel selection using intensity and/or local signal-to-noise criteria.

  2. Connected-component labeling (CCL) to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction.

3.1 Strong-pixel detection by local statistics

For each pixel \(i\) with value \(v_i\), consider a square window (nominally \(31\times 31\) pixels) around it. Let the window contain \(n\) valid pixels (excluding masked/bad/saturated), and define: \( \Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. \)

To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: \( \Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. \)

A variance-like quantity proportional to \(n'^2\) is formed: \( V = n'\Sigma_2' - (\Sigma')^2, \) and the deviation-from-mean quantity: \( \Delta = v_i n' - \Sigma'. \)

A pixel is considered strong if:

  • it is above a photon/count threshold, and

  • its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and

  • \(\Delta>0\), and

  • the squared deviation exceeds a scaled variance: \( \Delta^2 > V\cdot T^2, \) where \(T\) is the configured signal-to-noise threshold.

This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast.

The test is applied in two passes over the image. The first is as described above. The second repeats it with every pixel found strong by the first excluded from the local background — it is treated exactly like a saturated pixel, so it contributes to no window it falls into and stays strong itself. This matters for any spot wide enough to reach into its own background box: on a single pass such a spot inflates the mean and variance it is then tested against, and its outer pixels fail the criterion. Excluding the core recovers them, so the spot is reported with its true extent rather than its brightest few pixels. Both the CPU and GPU implementations run these two passes and return the same spot list for the same frame.

Special cases:

  • saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots),

  • invalid pixels are never strong.

3.2 Adaptive (self-calibrating) detection

The local-statistics test above needs a fixed photon/count threshold whose correct value depends on the background level, which varies between datasets. The adaptive mode (--adaptive-spots; the default in rugnux and in the viewer for both stills and rotation data, --no-adaptive-spots reverts) derives that threshold from each image’s own noise, per resolution ring, so no per-dataset value is needed. It admits more spots than the fixed threshold, including genuine reflections that belong to no indexed lattice; these are down-weighted rather than filtered in the per-image geometry fit (§7.4).

Pixels are binned into the same resolution rings as the azimuthal integrator (§2). For each ring a robust background is estimated in three passes: one plain pass over all valid pixels, then two \(\sigma\)-clipping passes that keep only pixels within \(\pm 3\sigma\) of the current ring mean (removing the Bragg peaks from the background estimate). This yields a per-ring background mean \(\mu_b\) and scatter \(\sigma_b\).

The ring’s detection threshold is the larger of two arms, \( t_b = \max\!\big(\;\mu_b + z\,\sqrt{\sigma_b^2 + \sigma_\mathrm{read}^2}\;,\;\; k_\mathrm{Poisson}(\mu_b, p)\;\big), \) where \(k_\mathrm{Poisson}(\mu_b,p)\) is the smallest count whose Poisson\((\mu_b)\) upper tail is \(\le p\). The Poisson arm is correct where the background is countable (a bright low-resolution ring gets a high threshold); the Gaussian arm — floored by a detector-level excess-noise constant \(\sigma_\mathrm{read}\) — takes over on near-empty high-resolution rings, where the Poisson arm degenerates to “one photon is significant” and would flood. The operating point \(p = E/N\) is set from a single portable knob \(E\), the expected number of false pixels tolerated per frame (--spot-false-pixels, default 100), with \(N\) the number of valid pixels. Because \(p\) and every \(\mu_b,\sigma_b\) come from the image itself, the same \(E\) lands a sensible photon threshold on strong and weak datasets alike, with no per-dataset tuning. Rings too sparse to characterise (fewer than ~40 pixels) fall back to a whole-frame background. A pixel is strong when \(v_i \ge t_b\) for its ring (saturated pixels are still forced strong); the strong pixels then feed the same CCL stage (§3.4). The signal-to-noise and photon-count criteria of §3.1 are not used in this mode.

Because detection reads the pixel’s ring, a pixel that falls outside the azimuthal-integration \(q\) range has no ring and can never be strong: the integration range bounds what adaptive detection can see. Both upper limits are therefore optional and default to the detector itself — the azimuthal integration runs to the highest \(q\) any pixel of the detector reaches (--azim-max-q unset), and spot finding is not clipped in resolution (--spot-high-resolution unset), for rotation data as well as stills. Setting either one narrows detection accordingly — appropriate for weak, high-background data, where the spots admitted at the detector edge are dominated by noise.

Fused GPU engine. The per-ring reduction the adaptive threshold needs is the same reduction the azimuthal integrator performs. On the GPU path the two are fused into a single image pass (AdaptiveSpotFinderGPU): one reduction accumulates the corrected per-ring sums for the azimuthal profile (§2) and the raw per-ring statistics for the threshold, after which a light kernel flags the strong pixels. One GPU pass therefore replaces both the separate azimuthal-integration pass and the host-side adaptive spot-finding pass, at a small fraction of the CPU finder’s cost per frame and producing the same spot list and azimuthal profile. It is enabled by default in the offline rugnux path, the interactive viewer and the online receiver.

Online. spot_finding_settings in the REST API carries adaptive_threshold and false_pixels_per_frame, so the mode is reachable from the broker and from the web frontend as well as from rugnux and the viewer. It defaults to off online, unlike rugnux and the viewer, because the broker serves both workflows and only one of them can run it: spots are found in software only on the DECTRIS/SIMPLON path, while the JUNGFRAU and EIGER workflows find them on the FPGA at its own fixed threshold. Setting adaptive_threshold on those is refused with an error rather than accepted and ignored, so a detection setting that had no effect cannot be mistaken for one that did.

3.3 Resolution and ice-ring handling

Spot finding can be restricted to a resolution range \([d_\mathrm{high}, d_\mathrm{low}]\) by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6).

A single per-image ice-ring score is derived from a radial profile: for each hexagonal-ice powder ring (positions \(d\) from Moreau et al., Acta Cryst D77, 2021), the profile intensity at the ring is divided by a smooth background estimated from the whole profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring’s ratio is reported (1 = no ice, \(>1\) = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring’s own edge (offline processing defaults to a fine 0.01 1/Å spacing, --azim-q-spacing, so the rings are well resolved). The reported quantity is the ice magnitude rather than a significance: with many photons any real ice ring is statistically significant, so significance does not discriminate.

The profile the score is read off is the peak-excluded one, not the plain azimuthal integration: where adaptive spot finding runs (§3.2 — the offline and viewer default), the score uses the sigma-clipped per-resolution-ring background that finder already computes for its threshold. This matters more than it sounds. A plain azimuthal profile is a per-ring mean, so a few strong low-resolution reflections landing in a ring’s bin raise it exactly as ice would; measured over a rotation battery, that alone ranked ice-free crystals above crystals that really are iced. An ice ring is azimuthally smooth and survives the sigma clip, while Bragg peaks do not, so on the clipped profile ice-free crystals sit near 1 and crystals with confirmed ice above 2. Only where no adaptive finder ran (the FPGA workflow) does the score fall back to the plain profile.

The radial profile sees ice only as a smooth powder ring. Ice in large crystallites diffracts as discrete spots, leaves the profile flat, and is invisible to the score above, so a second channel is read from the spot list itself: the spots found on the ice bands are counted against the spots found in the ice-free flanks \([w,2w)\) either side of each band, rescaled to the bands’ own \(q\) width (a flank landing on another ring is dropped with its width). The indicator is the ratio pooled over the run — per image the control is a handful of spots and the ratio means nothing — and it is taken before the spot-count filter, which orders ice spots last and would discard them first. The two channels barely overlap: smooth ice reads high on the profile and ~1 on the spots, textured ice the reverse, and a clean crystal ~1 on both. Both counts are stored per image (spot_count_ice_rings, spot_count_ice_control; HDF5 /entry/MX/peakCountIceRingRes, /entry/MX/peakCountIceRingControl).

Both channels are used offline as a gate on ice handling: unless the run reaches --ice-min-score (default 1.5) on the profile or --ice-min-spot-ratio (default 2.0) on the spots — 0 disables a channel — ice-ring flagging and the exclusion from the scale fit (§10.10) are skipped. The eleven fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss.

A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in \(1/d\) (or \(q\)) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low.

3.4 Connected-component labeling (CCL)

Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with:

  • centroid \((x,y)\) (often intensity-weighted),

  • pixel count (spot size),

  • integrated spot intensity proxy (sum of pixel values),

  • resolution \(d\) at the centroid (or mean over pixels),

  • and quality flags (e.g. ice-ring classification).

Spot-level filters include minimum/maximum pixel count and resolution limits.

The host implementation (StrongPixelSet::sparseccl) is the SparseCCL of the ACTS/traccc project: it runs over the strong pixels sorted row-major, uses a sliding window over the previous line and a union-find whose root is each component’s lowest index. On the GPU the same labelling runs on the device (SpotExtractorGPU): the packed strong-pixel bitmask is compacted into that same sorted list without leaving the card, each pixel finds its at most four earlier 8-neighbours by binary search, and a lock-free union-find with path halving labels them. Only the finished spot list — a few hundred entries — comes back to the host, instead of the whole bitmask (2.26 MB per frame at 18 MP). The two implementations produce the same components, in the same order, with the same pixel counts and intensities; tests/SpotExtractorGPUParityTest.cpp holds them to it. The device version is also insensitive to frame content: the host sliding window becomes quadratic when many pixels light up in one detector line — a hot module, or a diffraction ring where it runs tangent to a row — which costs tens to hundreds of milliseconds on such a frame, while the device version stays under a millisecond.

3.5 Adaptive per-image minimum spot size

The minimum-pixels-per-spot filter (§3.4) trades sensitivity against noise: a small value keeps faint one- or two-pixel spots — real signal on strong data, but detector noise on high-background frames — while a larger value keeps only well-formed spots. The best value is dataset-dependent, so for serial-stills indexing it can be chosen per image rather than fixed. The frame is indexed three times, at min-pix 3, 2 and 1, and the setting that maximises

\[ \frac{n_\mathrm{indexed}^2}{n_\mathrm{total}} \quad\text{(indexed-spot count weighted by indexed fraction)} \]

is kept; the frame is then integrated once at that min-pix. The fraction factor discounts the extra spots a smaller min-pix admits unless the lattice actually explains them, so strong frames keep their real weak spots (extending resolution) while noise-flooded frames stay strict. Because min-pix filters the connected components after detection, strong-pixel detection AND the connected-component labelling both run once per frame, and the three attempts only repeat the spot-level filter; the azimuthal profile is the one that single detection pass computed. The winning attempt’s spot list is kept rather than re-extracted, so the frame that is integrated is exactly the frame that was scored. This is a stills-only, indexing-path option — rotation indexing builds one global lattice from all frames and keeps a fixed min-pix. In rugnux it is the default; giving an explicit --min-pix-per-spot pins a fixed value instead.

4. Indexing overview

Indexing maps observed reciprocal-space vectors \(\mathbf{s}_i\) to a lattice such that: \( \mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, \) with integer \((h_i,k_i,l_i)\).

Jungfraujoch supports two complementary indexing strategies:

  1. FFT-based indexing (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples.

  2. Fast-feedback indexing (TORO-like): requires an approximate unit cell; optimized for speed and feedback.

Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return more than one lattice per image (multi-lattice indexing; see §5.4).

4.1 Indexed-spot decision (inlier test)

Given a trial lattice with direct basis vectors \(\mathbf{a},\mathbf{b},\mathbf{c}\) (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: \( h_f = \mathbf{s}\cdot\mathbf{a},\quad k_f = \mathbf{s}\cdot\mathbf{b},\quad l_f = \mathbf{s}\cdot\mathbf{c}. \) Let \((h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))\) and define the fractional residual: \( \delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. \) A spot is indexed if \(\delta^2 < \tau^2\), where \(\tau\) is the configured tolerance.

For indexed spots, the reciprocal lattice point \(\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*\) is used to compute \(\Delta_\mathrm{Ewald}(\mathbf{p})\) (stored as a diagnostic and later used in profile-radius estimation).

5. FFT indexing (unknown unit cell)

FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms.

5.1 Directional projections and histograms

Choose a set of unit vectors \(\{\mathbf{u}_d\}\) on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction \(d\), form a histogram in the scalar projection: \( t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. \)

Bin width is chosen approximately as: \( \Delta t \approx \frac{1}{2 L_\mathrm{max}}, \) where \(L_\mathrm{max}\) is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum \(q\) used (set by a high-resolution cutoff for indexing).

5.2 FFT peak picking and candidate vectors

For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along \(\mathbf{u}_d\). Each direction yields a candidate real-space length \(L\) chosen not by raw magnitude but by maximum prominence above a running-mean local background (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to \(L\ge L_\mathrm{min}\).

Candidate vectors are \(\mathbf{v}_d = L_d\,\mathbf{u}_d\).

A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak.

5.3 Lattice reduction and cell candidates

Triples of candidate vectors are combined to form candidate bases \((\mathbf{A},\mathbf{B},\mathbf{C})\), each reduced to its Niggli-reduced cell (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a widened fallback anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses.

5.4 Robust refinement and best-cell selection

Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked:

  1. more indexed spots wins — unless two candidates index within ~10 % of each other, in which case

  2. the smaller-volume cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then

  3. the smaller refinement score, then the spot count again.

Selection is not limited to a single lattice: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded.

An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant).

7. Geometry and lattice refinement

Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices.

7.1 Parameterization

The refinement jointly optimizes, depending on mode and constraints:

  • beam center \((x_\mathrm{beam}, y_\mathrm{beam})\),

  • detector distance \(D\),

  • detector tilt angles (two-angle model; third rotation often held at 0),

  • rotation axis direction (for rotation datasets),

  • crystal orientation (a global rotation),

  • unit-cell parameters, with constraints determined by inferred crystal system.

By default only the beam center, unit cell and crystal orientation are refined; the detector distance, tilt angles and rotation-axis direction are held fixed unless explicitly enabled. A lighter orientation-only mode refines just the crystal orientation, for stills whose geometry is already trusted. It carries a weak small-rotation prior penalising the whole angle-axis vector (all three components, at a low weight); what it is there for is the poorly-determined out-of-plane component, which is the one the data barely constrain.

For higher symmetries, constraints are enforced, e.g.

  • cubic: \(a=b=c,\ \alpha=\beta=\gamma=90^\circ\),

  • tetragonal: \(a=b\),

  • hexagonal: \(a=b,\ \gamma=120^\circ\),

  • monoclinic (unique axis \(b\)): \(\alpha=\gamma=90^\circ\), \(\beta\) refined.

7.2 Residuals and objective

For each indexed spot assigned integer \((h,k,l)\), compute:

  • observed reciprocal vector \(\mathbf{s}_\mathrm{obs}\) from its detector position and current geometry,

  • predicted reciprocal vector \(\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})\).

Residual is: \( \mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. \)

A non-linear least squares solver minimizes \(\sum \|\mathbf{r}\|^2\) over all selected inlier spots.

7.3 Rotation datasets: bringing observations to a common reference frame

For oscillation/rotation data, each image corresponds to a rotation angle \(\phi\) about an axis \(\mathbf{m}_2\). Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: \( \mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, \) with \(R(\phi)\) constructed from the axis-angle representation of the goniometer model. The angle \(\phi\) is taken at the centre of each frame’s oscillation (the frame angle plus half the oscillation width).

7.4 Multi-stage tightening of inlier tolerance

Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance \(0.3\to0.2\to0.1\)), which stabilizes convergence when starting from imperfect indexing and approximate geometry.

The loose first stage necessarily admits some spots that are not reflections of this lattice — the fraction of randomly placed spots inside a fractional-Miller tolerance \(t\) is \(\tfrac{4}{3}\pi t^3\), i.e. 11 % at \(t=0.3\) — and an unweighted fit lets them pull the orientation. Each residual is therefore weighted by how strong its spot is for its resolution: the frame’s spots are cut into equal-count resolution shells and each intensity is divided by its shell median, mapped to \(w^2=r/(1+r)\). The shell normalisation is what makes this safe — genuine high-resolution spots are legitimately weaker and carry the cell and distance information, so an un-normalised intensity weight would suppress exactly the spots the fit needs. The weight is a property of the spot and never of the current residual, so it does not depend on how far the geometry is from convergence.

7.5 Rotation geometry post-refinement (two-pass)

The refinement above (§7.2) runs per image against that image’s spots. For rotation data an additional post-refinement (on by default; --rotation-no-postrefine disables it) improves the detector distance, beam centre and crystal cell/axis using all frames at once, then re-integrates:

  1. Pass 1 integrates, scales and merges at the header geometry.

  2. From pass-1’s integrated reflections, the geometry is refined over all frames (Ceres, robust loss) in two separate steps rather than one joint fit:

    • Step A: crystal cell scale + goniometer-axis direction, from the observed rotation angles (a distance-independent excitation residual).

    • Step B: shared detector distance + beam centre, from the observed spot positions, with the cell held at step A — so the positional residual is no longer degenerate with the cell scale.

    Each step is cross-validated on a deterministic split of the reflections (an avalanche-mixed \(hkl\) hash, not a frame split and not an \(h+k+l\) parity, which would collide with a centering condition and leave the held-out half empty): fitted on one half, committed only if it lowers the held-out residual, otherwise left at nominal. The solver bounds the move — distance within ±5 %, beam centre within ±15 px — and detector tilt is held fixed, being gauge-coupled to the crystal orientation on a single crystal.

  3. Pass 2 re-indexes de novo and re-integrates at the committed geometry, reusing pass-1’s space group for the merge only. Only the detector distance and beam centre carry over: the refined cell and axis are used to make step B well-posed, but pass 2 re-indexes from scratch, so they are not propagated.

The refined pass is written as the canonical <prefix>_* output; the pass-1 (header-geometry) result is kept alongside as <prefix>_01_* for comparison.

Goniometer rotation scale (report only). A stage that turns further than it was commanded to leaves no trace in the file, because the stored \(\omega\) values are the commanded ones; the excess then presents as the crystal drifting, in this program and in others. Step A already measures it without a new degree of freedom: its residual rotates by \(-\phi\,\mathbf{u}\) with \(\mathbf{u}\) an unnormalised 3-vector, so \(|\mathbf{u}|\) is the factor by which the stage actually turned, and normalising the axis throws it away. It is reported, and warned about beyond 0.5 %, under the same cross-validation that gates the cell move — a fold that merely soaked up noise cannot raise the flag. It is a detector, not a calibration: nothing corrects the data, and it under-reads the true magnitude, because the fit only sees reflections that indexed at the nominal angle and per-frame orientation refinement has already absorbed part of the error.

7.6 Detector geometry from powder rings

Everything above fits the geometry to Bragg data, where the beam centre is the weakest parameter: it is gauge-coupled to the crystal orientation, which is why §7.5 restrains it toward the header value and commits only a sub-1 % move. A powder ring has no orientation to be coupled to. Where it falls on the detector depends on the geometry and on nothing else, which makes a calibrant — LaB₆, silver behenate, CeO₂, silicon — or even ice an independent constraint on exactly the quantity Bragg data cannot pin.

The ring positions are matched to the observed rings and the geometry is refined (Ceres, five parameters: beam centre, distance, and the two detector tilts) so that the \(|s|\) predicted at each observed ring point matches the ring it belongs to.

Calibrants. LaB₆, silver behenate, CeO₂ and silicon are held as unit cells and their rings enumerated from them. Ice is held as the eleven measured hexagonal-ice ring positions of §3.3 instead, because hexagonal ice is \(P6_3/mmc\) and enumerating \(hkl\) from its cell would emit rings that are systematically absent. A calibrant is therefore a list of ring \(q\) values throughout, not a cell.

What a ring can and cannot determine. A ring is a conic centred on the beam, so a wrong centre makes its apparent radius oscillate once per turn, \(r(\phi)=R+\delta_x\cos\phi+\delta_y\sin\phi\), with the same amplitude on every ring. A detector tilt \(\beta\) produces a \(\cos\phi\) term too — not the \(\cos2\phi\) one might expect — but one that grows as the ring’s radius squared, \(r(\phi)=R+(R^2/F)(\beta_x\cos\phi+\beta_y\sin\phi)\); the true \(\cos2\phi\) term is \(O(R^3\beta^2/F^2)\), hundredths of a pixel. The two are therefore separated by how the amplitude scales with radius, which needs at least two rings — on a single ring they are exactly degenerate. None of this uses the calibrant’s \(d\)-spacings, so the centre is determined without assuming anything about the standard.

The distance is different: it follows from \(r=F\tan2\theta\) with \(\sin\theta=\lambda/2d\), so a fractional error in the lattice constant passes straight into it, and the \(\lambda\)–\(F\) pair is separated only by the curvature of \(\tan(2\arcsin(\lambda/2d))\) across the rings — \(\partial\ln r/\partial\ln F=1\) at every ring against \(\partial\ln r/\partial\ln\lambda=4\tan\theta/\sin4\theta\), which runs from about 1.05 at low angle to 1.43 at high. That lever collapses as the detector moves back and the rings crowd into small \(2\theta\), so distance is a short-distance measurement and the wavelength is better calibrated by other means.

Reading the rings. The ring points come from one of two measurements, both accumulated over every processed image rather than one. The default reads the azimuthally-binned profile (§2) summed over the run: for each ring and each azimuthal sector, the radial peak is fitted against a locally interpolated background and the measured \((q,\phi)\) mapped back through the current geometry to the pixel it came from. The alternative pools the spot lists, which samples each arc wherever the spot finder’s threshold happens to bite. The accumulated profile is the same size however many images went into it; the pooled spot list is capped, each image contributing an equal share.

A plain radial profile — one azimuthal sector — has averaged the ring over every direction and carries no centre at all, so the profile route requires at least four sectors and uses 32 by default. Sixteen to thirty-two are enough; beyond that the limit is the ring’s own texture, not counting statistics.

The extraction window around a ring is capped at half the gap to its neighbour, because the background under a peak is taken from the ends of that window: hexagonal ice has three rings within 0.06 Å⁻¹ of one another, which a fixed window merges into a single peak. Where only one ring is in reach the two tilts are held at their input values rather than fitted, since on a single ring they are degenerate with the centre (above) and the fit would otherwise trade the centre away for them.

8. Reflection prediction

Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model.

8.1 Enumerating reciprocal lattice points

For a maximum resolution \(d_\mathrm{min}\), accept \((h,k,l)\) such that: \( \lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. \)

8.2 Still prediction (excitation-error cutoff)

For still images, the diffracting condition is approximated by an excitation-error cutoff: \( \left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. \) Accepted reflections are projected to the detector by intersecting the diffracted direction \(\mathbf{S}=\mathbf{S}_0+\mathbf{p}\) with the detector plane, using the current geometry.

When the beam has a finite energy bandwidth, this window is broadened radially per reflection: the cutoff is combined in quadrature with a bandwidth smear, \(\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}\), where \(\sigma_\mathrm{bw}\propto|p_z|\) (the reciprocal-space depth along the beam, growing as \(\sim 1/d^2\)). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same \(\sigma_\mathrm{bw}\) is deconvolved from the measured profile radius (§11.1), so it is not double-counted.

8.3 Rotation prediction (Laue equation + partiality model)

For rotation/oscillation datasets, Jungfraujoch solves for rotation angles \(\phi\) where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define:

  • rotation axis unit vector \(\mathbf{m}_2\),

  • \(\mathbf{S}_0\) incident vector,

  • \(\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)\).

A key quantity is: \( \zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad \mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, \) which also appears in XDS as the Lorentz component linked to the rotation axis.

A Gaussian mosaicity model yields a partiality fraction over an oscillation width \(\Delta\phi\):

\( P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], \)

with mosaicity \(\sigma_M\) in radians.

Reflections are predicted if they meet minimum \(\zeta\) and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area.

8.4 Systematic absences (centering)

Systematic absences are applied at the centering level (prior to full space-group symmetry) when the space group is supplied by the user. With no user-fixed space group, prediction runs in \(P\) regardless of the centering the lattice search inferred: the centering-absent reflections are integrated so that the space-group search (§13) can confirm or disprove the centering from the measured intensities, and so that a missed superstructure shows up. For centering symbol \(C\):

  • \(I\): absent if \(h+k+l\) odd,

  • \(A\): absent if \(k+l\) odd,

  • \(B\): absent if \(h+l\) odd,

  • \(C\): absent if \(h+k\) odd,

  • \(F\): absent if any of \(h+k, h+l, k+l\) is odd,

  • \(R\): absent if \((-h+k+l)\bmod 3 \ne 0\),

  • \(P\): no centering absences.

9. 2D Bragg integration (profile fitting over a three-ring ROI)

Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The default extraction is profile fitting (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection \((I,\sigma,\text{partiality},d)\), so scaling, the rotation combine (§10.6) and merging consume either unchanged.

9.1 Regions of interest

For each predicted reflection at \((x_p,y_p)\), define three radii:

  • \(r_1\): inner signal radius,

  • \(r_2\): inner background radius,

  • \(r_3\): outer background radius.

Pixels are classified by their squared distance \(r^2=(x-x_p)^2+(y-y_p)^2\):

  • signal region: \(r^2 < r_1^2\),

  • background annulus: \(r_2^2 \le r^2 < r_3^2\).

Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk (\(r<r_2\)) of any other predicted reflection are removed from this reflection’s background annulus, so a neighbouring spot cannot leak into the background estimate. (Both the annulus and that exclusion become ellipses when the option below is used; with it off, which is the default, they are the circles just described.)

Radially elongated background ring (opt-in, --integration-stencil <k>, default 0). The three radii above are fixed pixel counts, identical for every reflection at every resolution. A reflection is not round, though: a finite bandwidth streaks it radially by \(\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}\). On a radially smeared spot the fixed \(6\ldots13\) px ring therefore sits only \(\approx1.3\)–\(2.2\) radial \(\sigma\) from the centre — on the reflection’s own tails, which it then measures as background.

With \(k>0\) the background ring becomes an ellipse, elongated along the beam→reflection direction by \(k\sigma_\mathrm{bw}\). The radial semi-axes become \(r_2+k\sigma_\mathrm{bw}\) and \(r_3+k\sigma_\mathrm{bw}\); the tangential half-widths stay \(r_2\) and \(r_3\); and the growth is capped at \(2r_3\), which bounds what a mis-declared bandwidth can do to the bounding box. Pixels are then classified as

  • signal region: \(r^2 < r_1^2\) — a circle, unchanged,

  • background ring: \(r^2-q_\mathrm{in}\rho^2 \ge r_2^2\) and \(r^2-q_\mathrm{out}\rho^2 < r_3^2\),

where \(\rho\) is the pixel’s radial offset (its projection on the beam→reflection direction), \(g=\min(k\sigma_\mathrm{bw},\,2r_3)\) is the capped growth, and \(q=1-\big(r/(r+g)\big)^2\) for the boundary concerned. Written this way \(k=0\) gives \(q=0\) and both tests collapse onto \(r^2\) exactly in floating point, so the default classifies every pixel exactly as the circular stencil did. The neighbour exclusion above follows: each neighbour’s inner ellipse, taken in that neighbour’s own radial frame, is what is masked out of this reflection’s ring.

The width is the bandwidth streak alone, and deliberately not the profile’s full radial variance of §9.3, which also carries the sensor parallax and weak-spot capture terms. Those two are the only terms there are on a monochromatic beam, and widening the ring by them was measured on the rotation battery: it neither helped the crystals with clean high-resolution shells nor left the weak ones alone. The bandwidth streak, by contrast, is a measured elongation of the recorded spot — principal axis along the radius to within a couple of degrees, and azimuth-independent. Keeping only it also makes the option exactly inert on a monochromatic beam, where \(\sigma_\mathrm{bw}\) is zero.

Growing the ring also grows the neighbour exclusion, so on a crowded pattern fewer background pixels survive; a reflection left with too few is rejected outright. On the data this was measured on the loss is under 0.1% of reflections, but it is not structurally zero.

Only the ring moves. The signal disk \(r_1\) stays circular, deliberately: it sets \(n_S\) and it sets \(\mathrm{var}(\hat b)\), and with --integrator boxsum it drives the all-or-nothing “every signal pixel valid” acceptance gate (§9.2), so growing it would reject any box sum carrying a single bad pixel anywhere along a long streak. What a circular \(r_1\) loses is flux, and that loss is a function of resolution alone, which the per-shell scale absorbs.

9.2 Box summation (seed and fallback)

Let:

  • \(S = \sum I(x,y)\) over signal pixels,

  • \(n_S\) = number of valid signal pixels,

  • \(B = \sum I(x,y)\) over background pixels,

  • \(n_B\) = number of valid background pixels.

Background per pixel and integrated intensity: \( \hat{b} = \frac{B}{n_B},\qquad \hat{I} = S - n_S \hat{b}, \) with a Poisson-like uncertainty \(\sigma(\hat{I})=\max\!\big(1,\ r_\sigma\hat{I},\ \sqrt{S + n_S^2\,\mathrm{var}(\hat{b})}\big)\), i.e. \(\sqrt{S}\) floored both at 1 and at a small fraction \(r_\sigma\) of the intensity. The second term under the root is the uncertainty of the background estimate itself: \(\hat b\) is measured from a finite number of ring pixels, \(\mathrm{var}(\hat b)=\hat b/n_B\), and it is subtracted \(n_S\) times over, so it enters squared. Omitting it understates \(\sigma\) by \(\sqrt{1+n_S/n_B}\) — 1.109 with the shipped circular stencil; with an elongated ring \(n_B\) grows with resolution, so the factor is no longer one number for a run — uniformly, on every reflection of every dataset. The same term is carried into the profile fit (§9.3), where it adds \((\sum wP/\sum P^2/v)^2\,\mathrm{var}(\hat b)\); \(n_B\) is the count of pixels behind the final background value, so a clip or trim that discards ring pixels raises it. A box sum is accepted as “observed” only if all signal pixels were valid and \(n_B\) exceeds a minimum — it measures what is in the disk with no model of what should be there, so it cannot renormalise a disk it has lost pixels out of. The profile modes can, and do (§9.3). This box sum is the classical estimator; it is used directly with --integrator boxsum, and otherwise seeds the profile fit below, where \(S\) and \(n_S\) then count only the pixels that were actually read.

High-side clipped background (default on). Because \(\hat{I}=S-n_S\hat{b}\) is a small difference of large numbers for weak reflections, a per-pixel background bias \(\delta\hat{b}\) becomes a fractional intensity bias \(\approx n_S\,\delta\hat{b}/\hat{I}\) that grows as \(\hat{I}\) shrinks — worst at the resolution edge. A plain ring mean reads high there, because neighbour-spot wings that survive the signal-disk mask, tails and zingers are one-sided (positive) contaminants. The ring mean is therefore made robust: pixels above \(\hat{b}+n\sqrt{\hat{b}}\) are rejected and the mean recomputed, with \(n=4\) (--background-clip; \(n=0\) disables), lowered by rugnux to \(n=3\) on broadband (non-zero bandwidth: pink-beam / DMM) data, where a bandwidth-streaked high-resolution spot leaks into the ring more readily. That is only a default — the flag sets \(n\) whatever the bandwidth is. A clean Poisson ring is essentially unchanged by the cut (measured false-rejection rate 0.04–0.39 % at \(4\sigma\)), while a 40-pixel neighbour core at \(+100\) counts shifts the estimate by \(+0.009\) ct/px.

The clip cuts only the high tail, which matters: the symmetric trimmed mean it replaced (drop the lowest and highest fraction \(f\) of ring pixels, \(f=0.10\); still reachable with --background-trim, which switches the clip off) is not a consistent estimator of the mean of a right-skewed Poisson sample. It sits \(\approx0.1\) ct/px below the true mean at every level, and with \(n_S\approx50\) signal pixels in the \(r_1\) disk that under-estimate adds \(\approx5\) counts to every partial — negligible at low resolution, but a large fraction of a partial in the outermost shell. The trim also collapses once contamination exceeds \(\approx10\,\%\) of the ring, where the clip does not. Note that removing a positive background bias lowers \(\langle I/\sigma\rangle\) and raises edge \(R_\text{meas}\), because both are inflated by information-free counts — so neither may be read as evidence against the change.

Both estimators are computed in the shared background pass, but only the trim reaches plain box summation: the high-side clip is skipped for --integrator boxsum, which therefore uses the plain ring mean unless --background-trim is given.

Radial background correction (opt-in). A ring mean estimates the background under the signal disk correctly only if the background is flat there. The signal disk and the ring are concentric, so for a background that is linear in position \(\langle B\rangle_\mathrm{ring}=\langle B\rangle_\mathrm{disk}\) identically — a plane or gradient fit buys exactly nothing. The leading error is the curvature of the radial background, which is negligible on a smooth background but reaches tens of counts on a single reflection sitting on a sharp powder ring. That error is a kernel over radial offset, \( \delta \hat b \;=\; \textstyle\sum_k \kappa_k\, \bar B(r_0+k), \) with \(\kappa\) the annulus-minus-disk histogram of the stencil over radial offset, averaged over azimuth, and \(\bar B(r)\) the image’s own radial background curve. With the fixed circular stencil (\(k=0\), §9.1) that single kernel serves every reflection. An elongated ring does not: its radial-offset histogram depends on how far that particular reflection’s ring was grown, so \(\kappa\) becomes a small table of kernels, indexed by the growth rounded to whole pixels. The azimuthal average survives the change unaltered, because the stencil is rebuilt in the reflection’s own frame at each azimuth and so stays radially aligned: what is averaged over is the sub-pixel phase of the detector grid against the radius, which is what genuinely differs between reflections. Applying it costs one short dot product per reflection and no extra pixel reads; correcting the background scalar means the box sum, the profile fit and the variance all pick it up. The curve is accumulated from the same annulus pixels the background pass already reads (a pixel’s radius is the reflection’s radius plus the pixel’s projection on the beam→reflection direction, so no per-pixel square root is needed) and specifically from the clipped pixels, or it would carry neighbour tails and zingers — which is why the correction is inert under --integrator boxsum, that path having no clip pass.

The model is a function of radius alone, so it is applied only where that is true of the background. --background-radial takes on, off or auto. It is off by default; under auto each image’s peak-excluded ice score (§3.3) is taken after spot detection and before integration, and the correction is applied to that image when the score reaches the same --ice-min-score gate. Smooth powder ice is a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes 43 % of the bands’ excess amplitude, and the improvement is 7× larger inside the bands than outside, which is its stated mechanism; on a crystal whose ice is textured the same correction increased the excess amplitude by half; on a clean crystal it is inert to four decimal places. Auto engages only where a peak-excluded score exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off.

9.3 Profile-fitted extraction (default)

A fixed signal disk captures a width-dependent fraction of each spot, which puts a multiplicative floor on the per-observation precision of strong reflections and weights weak reflections poorly. Profile fitting removes this by extracting each intensity against a fitted spot shape, without needing reference intensities. Per frame:

  1. Seed. Box-sum every reflection (§9.2) to get a rough intensity and observed centroid, and select strong spots (significance \(\ge 5\)).

  2. Build the profile. For gaussian (the default) the width is taken per resolution shell from the measured second moments of the strong spots (shell-dependent because spot size grows with resolution). The moments are anisotropic: each strong spot’s pixels are rotated into its own radial/tangential frame before being accumulated, giving \(\sigma^2_r\) and \(\sigma^2_t\) separately. Stacking the spots in the detector frame instead — they sit at every azimuth — averages the two directions away, leaving only \(\sigma_r^2+\sigma_t^2\), so radial smearing is read back as a wider tangential spot. For empirical the profile is instead the averaged, background-subtracted pixel grid of the shell’s strong spots, accumulated in the detector frame on their rounded predicted positions. For gaussian only, the profile is then rebuilt for each reflection, centred on its sub-pixel predicted position (the noise-free geometric centre, not the observed centroid) and, where needed, elongated only along the radial direction (away from the beam centre) — because two effects stretch a spot radially but not tangentially:

    • a finite energy bandwidth smears each spot by \(\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}\) (\(R_\mathrm{px}\) = distance from the beam centre, large at high resolution), and

    • sensor parallax — the depth over which a photon converts in a thick Si/CdTe sensor — adds a term \(\propto\tan^2(2\theta)\) (material- and energy-dependent), plus a small fixed weak-spot capture term.

    The two enter as a floor on the measured radial excess: \(\sigma^2_\mathrm{radial}=\sigma^2_t+\max\!\left(\sigma^2_r-\sigma^2_t,\ \sigma_\mathrm{bw}^2+c_\mathrm{par}\tan^2(2\theta)\right)\), tangential unchanged at \(\sigma^2_t\). The measured excess is what the signal disk can resolve; the analytic term takes over for a streak too long to be measured there. The Gaussian is built on a grid grown to hold the streak — capturing it without the tangential background an isotropic widening would add. The empirical profile keeps the fixed per-shell grid and gets none of this.

  3. Fit (Kabsch). With profile \(P\), background \(B\) and the shell variance model, the intensity and its uncertainty are \( I = \frac{\sum P\,(c-B)/v}{\sum P^2/v},\qquad \sigma = \sqrt{\frac{1}{\sum P^2/v}},\qquad v = \max\!\left(B + I\,P,\ \tfrac{1}{2}B\right), \) where \(c\) is the pixel value and the de-biased variance \(v\) (background plus model signal, rather than the down-fluctuating observed count) is iterated (a few passes). The plug-in \(I\) enters as it is: half-wave rectifying it, \(v=B+\max(I,0)P\), lets \(v\) — and with it the reported \(1/\sum P^2/v\) — respond only to upward fluctuations of a noisy estimate, which adds \(\approx0.4\,\sigma\sum P^3/(\sum P^2)^2\) to every \(\sigma\) whatever the count rate. That offset is invisible on strong reflections and a large fractional inflation on weak ones; the \(\tfrac12 B\) clamp keeps \(v\) positive without reintroducing it. As a guard, if the profile intensity runs away from the box-sum seed (by more than ~10 box-sum \(\sigma\)) it falls back to the seed, and the background term is floored at \(0.01\) ct/px — enough to keep \(P^2/v\) finite when the ring mean reads exactly zero, which a ring of \(n_B\) pixels cannot distinguish from any background below \(\approx1/n_B\). The rotation/excitation partiality is carried exactly as in the box-sum path.

Pixels the fit cannot use (MINPK). A profile fit is the amplitude of a normalised profile, so a pixel left out of the sum renormalises the estimator by construction: it costs information — \(\sum P^2/v\) shrinks and \(\sigma\) grows — but biases nothing. That is what keeps a reflection whose signal disk is cut by a mask, an untrusted region, a detector gap or an overload: those pixels are simply not read, and the fit is taken over the rest, exactly as the shared pixels of a crowded reflection are (--overlap exclude). The reflection is kept only while enough of the expected profile survives — at least --overlap-minpk of the profile mass that falls on the detector at all, default 0.75, which is XDS’s MINPK and dials’ valid_foreground_threshold. The complete reflections alone teach the profile, its resolution shells and their widths. --integrator boxsum has no profile to renormalise with and keeps the all-or-nothing rule of §9.2.

“Biases nothing” holds only while the profile model is exact. Lose the peak and the amplitude is set by the wings alone, so the result stops being a measurement of the reflection and becomes a measurement of how well the fitted shape describes it. The worst case is a pixel invalidated by the flux it saw — a detector’s per-frame overload marker: that pixel goes missing because the reflection was bright, so the loss concentrates on the strong low-resolution reflections that are the largest terms of \(R_\mathrm{meas}\), where the fit reads \(-50\%\) against the symmetry mates. MINPK cannot catch it, because it cuts on profile mass and the peak of a broad spot is a few percent of the mass. So a second condition applies alongside it, on any unreadable pixel whatever made it unreadable: no unreadable pixel may carry more than 0.9 of the profile’s own peak value. As a fraction of the peak rather than a radius in pixels, that scales with the spot — for a Gaussian it is a cut at \(\sqrt{-2\ln f}\,\sigma = 0.46\sigma\), the peak pixel alone where \(\sigma\) is 0.8 px and the crest of the ridge where the profile is a bandwidth streak — and it costs well under 0.1 % of the recovered observations.

The integrator is selected by --integrator boxsum|gaussian|empirical (default gaussian).

9.4 Lorentz–polarization factor handling

For integrated reflections, polarization correction can be applied as a multiplicative correction to the reflection scale via the geometry-based polarization term (§2.2). A Lorentz-like factor is carried as rlp in predictions, and used during scaling/merging (§10).

10. Scaling and merging

After per-image integration, Jungfraujoch scales observations and merges them into unique reflections. The design is intentionally compatible with XDS/XSCALE concepts, and handles both still and rotation data.

10.1 Observation model

For an observation \(j\) of a unique reflection \(h\) on image (or image group) \(i\), the predicted measured intensity is modeled as: \( I_{ij} \approx G_i \, L_{ij}\, P_{ij}\, I_h, \) where:

  • \(G_i\) is the image scale factor,

  • \(L_{ij}\) is a Lorentz-like / geometry factor; predictions carry its reciprocal as rlp, so \(L = 1/\texttt{rlp}\) and the correction below is applied as a multiplication by rlp,

  • \(P_{ij}\) is a partiality term (model-dependent),

  • \(I_h\) is the merged (true) intensity parameter for that unique reflection.

A least-squares objective is minimized: \( \sum_{ij} \left(\frac{I_{ij}^{\mathrm{pred}} - I_{ij}^{\mathrm{obs}}}{\sigma_{ij}}\right)^2 \) solved by robust (Cauchy) weighted least squares, with optional post-fit smoothing of the per-frame scales for rotation series (§10.3).

10.2 Partiality models

The partiality applied is fixed by the data type and scaling stage, not chosen from a user menu:

  1. Rotation partiality (XDS-like; see §8.3), used for the per-frame scaling of rotation partials: \( P_{ij} = \frac{1}{2}\left[ \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) \right]. \) The mosaicity \(\sigma_{M,i}\) is measured once per image at indexing (MLE, §11.2) and held fixed during scaling — only smoothed in frame order (§10.3), never re-refined (it is degenerate with the scale \(G\); §11.2).

  2. Unity (\(P_{ij}=1\)): used for the scale-on-fulls refit (§10.6), where each observation is already a complete reflection.

  3. Fixed: use the per-reflection partiality carried from prediction. Still/serial images are predicted with \(P=1\), so a single-pass stills scale is effectively unity/fixed — which is exactly what --simple-stills keeps. By default the stills path instead post-refines a physical partiality: a small per-crystal orientation tilt \((\delta\psi_x,\delta\psi_y)\) about the two axes perpendicular to the beam is refined against the running merge, and every reflection’s partiality is then recomputed analytically from the refined lattice through its excitation error \(\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|\) and a Gaussian width \(\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2\) — the reciprocal-lattice point’s own radius (resolution-independent), the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature. The fit typically drives \(\gamma_e\to0\), leaving the resolution-independent \(\gamma_0\) as the effective width. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the spatial pattern of partialities — a degree of freedom the per-image scale \(G\) does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with \(G\). Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with \(G\) profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute \(P\) → re-merge, repeated a few times.

Reflections below a minimum partiality can be rejected from merging to avoid unstable corrections.

10.3 Smoothing of per-frame scales

The per-frame scales \(G_i\) are fit by robust (Cauchy) inverse-variance-weighted ratios; there is no explicit \(G\approx1\) prior. For rotation datasets, optional smoothing enforces the expectation that scale and mosaicity vary slowly across a sweep: after the per-frame fit, \(\log G_i\) (and the mosaicity) are replaced by a centred moving average over a window spanning a configurable rotation range (XDS DELPHI-like; --smooth-g, default 5° for rot3d, off otherwise). It is a post-fit smoothing pass, not a curvature penalty inside the least-squares objective.

The crystal orientation is smoothed the same way, and for the same reason. Geometry is re-refined independently on every frame against that frame’s spots alone — as few as a dozen on a sparse crystal — so the per-frame orientation carries a real slow drift (crystal slippage, up to ~1.3° across a sweep) on top of fit noise that scales with spots per frame. Before scaling, the per-frame lattices are de-rotated to a common reference, averaged in frame order, rotated back, and every partial’s \(\Delta\phi\) — hence its partiality — is recomputed from the smoothed lattice. The window is chosen per dataset by leave-one-out cross-validation (does a frame’s neighbours predict its orientation?) rather than fixed, because drift and noise both vary by two orders of magnitude between crystals; it is capped, because the per-frame fit also absorbs a real per-frame systematic that smoothing too wide destroys. Only frames that actually indexed take part: a frame that did not carries an all-zero lattice, which is finite and so passes a validity check written as a finite test, and would otherwise be both averaged into its neighbours’ orientation and scored in the cross-validation that picks the window. Refining less is not an alternative: with per-image refinement off the space group is lost on several crystals.

A per-frame scale enters every intensity as \(1/G\), so a frame whose fit is not determined by its data can amplify it without bound — and \(\sigma\) is amplified by the same factor, which makes it invisible to any \(\sigma\)-based outlier test. A fitted \(G\) far below the run’s median is therefore treated as undetermined rather than as a successful fit, both here and in the separate refit on the combined fulls (§10.6). The bound is a ratio to the run’s own median because \(G\) is not gauge-fixed: it and the merged means have an exact global multiplicative degeneracy, so no absolute value is meaningful.

10.4 Merging estimator

After refinement, corrected observations are formed: \( I^{\mathrm{corr}}_{ij} = \frac{I^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}},\qquad \sigma^{\mathrm{corr}}_{ij} = \frac{\sigma^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}}. \)

Unique intensities are merged by inverse-variance weighted mean: \( I_h = \frac{\sum_j w_j I^{\mathrm{corr}}_{ij}}{\sum_j w_j},\qquad w_j = \frac{1}{(\sigma^{\mathrm{corr}}_{ij})^2}. \)

The weights use an expected variance: the Poisson signal part of each \(\sigma^{\mathrm{corr}}_{ij}\) is rebuilt at the reflection’s merged \(\langle I\rangle\) rather than at that observation’s own intensity. Weighting by an observation’s own \(\sigma^2\) biases the inverse-variance mean low below about one photon, because an up-fluctuated observation gets a larger sigma and is then down-weighted too hard. The rotation combine already does this; for stills it is on by default, and --no-expected-variance-merge restores the observed-sigma weighting.

An internal-consistency term can inflate uncertainties when multiple observations are present, in the spirit of XSCALE.

10.5 Merging statistics

Per-shell and overall merging statistics are computed on corrected intensities, including:

  • number of observations and of unique reflections, and multiplicity,

  • mean \(I/\sigma(I)\),

  • \(R_\mathrm{meas}\) (the redundancy-independent Diederichs–Karplus form) from within‑HKL deviations,

  • \(\mathrm{CC}_{1/2}\) (half-set correlation) and, when a reference dataset is supplied, \(\mathrm{CC}_\mathrm{ref}\),

  • completeness against the enumerated reflections for the cell and symmetry,

  • the anomalous signal-to-noise \(\mathrm{SigAno}\) (below).

The error model is refined as \(\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2\), with \(a\) set by the scatter of weak (counting-limited) reflections and \(b\) the intensity-proportional systematic scatter of the strong ones. On the rotation path, ISa is the asymptotic (\(I\to\infty\)) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, Acta Cryst. D66 (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the \(I/\sigma\) threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as \(1/b\) of the whole-range fit, whose \(b\) is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is report-only — nothing downstream reads it, and the merged \(\sigma\) is not floored at \(b|I|\) (that floor was removed). The per-observation \(\sigma_\mathrm{corr}\) (the merge weights) uses the whole-range \(a,b\). The stills path has no asymptotic estimate and reports \(\mathrm{ISa}=1/b\) directly.

\(a\) and \(b\) are reported in XDS’s convention, which is \(\sigma^2 = a(\sigma_0^2 + b I^2)\) with \(\mathrm{ISa}=1/\sqrt{ab}\), so the printed pair can be read straight against a CORRECT.LP. The internal fit keeps the form above; only the report converts, as \(b_\mathrm{XDS} = b^2/a\). Note that \(a\) is the same in both conventions and that the two ISa expressions are the same number, \(1/\sqrt{a\cdot b^2/a} = 1/b\) — so the rotation log prints two ISa, the whole-range \(1/b\) (XDS’s meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: _reflns.jfjoch_diffrn_ISa is the whole-range value, directly comparable with a CORRECT.LP, and the asymptote is written separately as _reflns.jfjoch_diffrn_ISa_asymptotic, with _reflns.jfjoch_error_model_a and _b alongside so the number can be re-derived. Note that a file written before this change carries the asymptote under the plain ISa name. A third, unrelated \(b\) appears in the space-group search (§13.1); it is fitted with the \(\sigma^2\) coefficient held at 1 and its gate constants are calibrated in that convention.

Anomalous signal-to-noise (SigAno). The strength of the anomalous signal is reported per shell and overall as \(\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle\), where \(\Delta I = I(+)-I(-)\) over acentric reflections measured in both Bijvoet hands and \(\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}\). It is computed from the full-multiplicity inverse-variance \(I(+)/I(-)\) split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise \(\mathrm{SigAno}\) approaches the half-normal value \(\sqrt{2/\pi}\approx0.8\), and it rises above \(1\) once a real anomalous difference is present. A half-set anomalous correlation (“\(\mathrm{CC}_\mathrm{anom}\)”) is not reported. Its two half estimates \(\Delta I_0,\Delta I_1\) are complementary partitions of one observation pool (\(\Delta I_0+\Delta I_1=2\,\Delta I_\mathrm{full}\)), and subtracting the two Bijvoet hands cancels the large common intensity that keeps \(\mathrm{CC}_{1/2}\) non-negative, leaving the small anomalous signal against the per-half split noise; below an anomalous signal-to-noise of \(1\) per half that correlation tends towards \(-1\) rather than \(0\). \(\mathrm{SigAno}\) has no such floor. It is emitted only when an anomalous split was made, using the standard PDBx items _reflns.pdbx_absDiff_over_sigma_anomalous (overall) and _reflns_shell.pdbx_absDiff_over_sigma_anomalous (per shell), and appears as the SigAno column of the printed merge-statistics table.

10.6 Rotation datasets: combining partials into fulls (3D integration)

In a rotation scan a reflection is recorded as a series of partials spread across the frames its rocking curve crosses. Merging those partials directly would force the merge error model to absorb the rocking-curve slicing as if it were measurement noise, capping the achievable \(I/\sigma\). For rotation data Jungfraujoch instead combines each reflection’s partials into a single full intensity first, then scales and merges the fulls — a 3D integration over the rocking curve.

The combine groups each reflection’s partials into rocking events (contiguous runs of frames) and reduces each event to one full:

  • De-biased weighted sum. Partials are combined by inverse-variance weighting, where each partial’s variance is its background-noise component plus the model signal shared across the event (Kabsch profile-fit form). Using the shared model signal rather than the individual down-fluctuating intensity stops weak partials from being over-weighted, which would otherwise inflate the merged error model. The weights depend on the full, so the estimate is iterated.

  • Captured fraction. The partiality summed over the event, \(f=\min(1,\sum_j p_j)\), measures how completely the rocking curve was sampled. A full whose curve was captured below a threshold (--min-captured-fraction, default 0.7 for rotation) is dropped — an event seen over only a small fraction of its curve is unreliable however many frames it spans. (The per-partial minimum-partiality cut of §10.2 still applies upstream, in the per-frame scaling.)

  • Per-image rejection (opt-in). A frame whose observations correlate poorly with the merged reference is not measuring the crystal being merged — it may be off-crystal, or on a different crystal where two lattices occupy separate regions of the sample. --min-image-cc drops such frames. It has no default: the per-frame correlation measures data quality as much as frame validity, and its typical level varies widely between datasets, so no single absolute bound is generally valid.

  • Capture-aware uncertainty. A full captured incompletely (\(f<1\)) is extrapolated and biased high. The unobserved fraction is charged as an extra systematic uncertainty, \(\sigma^2 \leftarrow \sigma^2 + \big(c\,(1-f)\,I\big)^2\), so the merge down-weights these extrapolated fulls and the error model treats their scatter as expected. It is enabled by default for the rotation path.

The fulls are then re-scaled in the XDS sense — a per-image scale refit directly on the complete reflections under the unity partiality model — and merged (§10.4). Because every merged observation is now a counting-statistics-limited full rather than a partiality-divided slice, the error model reaches a far higher asymptotic \(I/\sigma\).

After scale-fulls, four correction surfaces are fitted on the combined fulls (rotation path, on by default; disable all with --no-scaling-corrections), each an alternating multiplicative refinement of the per-full scale against the merged reference:

  • Decay. Radiation damage weakens later frames more at higher resolution — a resolution×time (Debye–Waller) systematic the resolution-flat per-image scale cannot capture. A single global relative-\(B\) rate is fitted, \(\ln(I_\mathrm{ref}/I_\mathrm{obs}) = 2\,(\mathrm{d}B/\mathrm{d}n)\,(n-\bar n)\,s^2\) (frame \(n\), \(s^2 = 1/4d^2\)), and folded into the scale. It engages only when the total relative-\(B\) over the run exceeds a physical floor (2 Ų); below that the decay is negligible and “correcting” it only spreads symmetry equivalents (same \(s^2\), different frames). An optional per-batch relative-\(B\) (--relative-b[=deg], off unless requested; 10°-of-rotation batches by default) extends the single global rate to a smooth \(B(n)\) curve — the same \(s^2\)-weighted decay fit solved independently over short frame batches, curvature-penalized so it cannot over-fit and cross-validated like the surfaces below — for crystals whose decay is non-linear in dose. Its cross-validation splits on ASU-group parity, not the frame parity the surfaces below use: a per-batch parameter owns whole frames and so cannot be scored on a held-out frame, whereas splitting the symmetry equivalents tests whether a batch’s \(B\) generalises to reflections it was not fitted on.

  • Absorption. A smooth multiplicative factor over the diffracted-beam direction expressed in the goniometer (crystal) frame: each full’s predicted detector position gives the lab diffracted direction, de-rotated by the spindle so a fixed crystal-frame direction is sampled at many rotation angles and its grid cell is well-determined. Negligible at hard X-rays / thin crystals; it matters at low photon energy.

  • Modulation (detector-plane flat-field). A smooth multiplicative factor over where each reflection lands on the detector (predicted \(x,y\)): symmetry-equivalents land at different positions as the crystal rotates, over-determining the surface. It absorbs detector-response and geometric systematics that inflate \(R_\mathrm{meas}\).

  • Time-dependent absorption. The same surface as Absorption, but over (rotation angle × detector position) instead of the crystal-frame direction alone. The two agree while the illuminated volume stays put — the incident path then depends only on the spindle angle, which the per-image scale already takes, and the exit path is fixed in the crystal frame. Once the diffracting volume drifts through the beam the exit path becomes a function of the spindle angle as well, and nothing time-independent describes it. Fitted last, on 12 rotation bins × a 10×10 equal-occupancy detector grid, so the two time-independent surfaces get first claim on what they can explain.

Each surface is cross-validated: fitted on even-numbered frames and kept only if it improves the held-out odd-frame agreement by a clear margin (and vice versa), scored by a σ-independent, \(R_\mathrm{meas}\)-like fractional agreement \(\sum|I_s-I_\mathrm{ref}|/\sum|I_\mathrm{ref}|\) rather than a studentized \(\chi^2\) — so a surface cannot “pass” by reshaping the sigmas instead of tightening the intensities. A surface fitted to noise where its systematic is absent does not generalize and is discarded — a correction never adds scatter.

Radiation-damage report (rotation, report-only). Independently of whether any decay correction is applied, rugnux measures and reports the relative Debye–Waller \(B\) across the sweep: the per-image scale’s correlation to the merge and the per-image mosaicity versus frame (dose), together with a per-batch relative-\(B\) curve whose first→last change is a single headline number (measured before any decay correction, against the least-damaged early wedge). It is written to the log and to the merged mmCIF as a data-quality-vs-dose diagnostic and never alters the merged intensities — distinct from the decay correction above, which does fold into the scale.

Each batch’s \(B\) is fitted on resolution-shell means, not on single observations: \(\ln(I_\mathrm{ref}/I_\mathrm{obs})\) of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs \(I_\mathrm{obs} > 0\), which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely dimmer than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as absent rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report (docs/RUGNUX.md) to name.

10.7 R-free test-set flags

A fraction of the unique reflections (rfree_fraction, default 0.05) is flagged as a free (test) set, written to the output (MTZ FreeR_flag, mmCIF _refln.status_free, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection’s Friedel-merged (Laue) ASU index, which gives three properties:

  • all symmetry- and Friedel-equivalent reflections share one flag — in particular a Bijvoet pair \(I(+)/I(-)\), kept as two separate merged rows in anomalous mode, is never split across the work and free sets (which would bias R-free);

  • the free/work decision is a deterministic hash of that key, so the same reflection always lands in the same set — reproducible run-to-run and independent of the order in which observations were merged;

  • the hash depends only on the reflection index, not on this dataset’s resolution range or which reflections it happens to contain, so a uniform draw takes ~rfree_fraction of the distinct reflections free and — crucially — every dataset of one crystal form gets the same free set. That cross-dataset consistency is what a multi-dataset campaign (ensemble refinement, PanDDA) requires; a per-shell stratification tied to each dataset’s own \(d_\mathrm{min}\) would break it.

On small data, where rfree_fraction (default 0.05) would give too few test reflections for a statistically stable R-free (Brünger’s ~500–2000 rule), the fraction is floored so at least ~500 distinct reflections are free — capped at 10 % so a large test set never steals working data. For ordinary data this floor is inactive and the fraction stays flat at rfree_fraction, preserving the cross-dataset-identical property above; it only lifts the fraction on genuinely small datasets, where per-dataset R-free stability outweighs cross-dataset identity (and a shared reference set is the way to keep exact identity there).

When a reference MTZ (--reference-mtz) carries a FreeR_flag column, its test set is imported instead: every merged reflection whose Laue-ASU index matches the reference takes the reference’s flag (reflections absent from the reference keep the hash flag). This lets a whole fragment-screening campaign inherit one shared free set from the apo/reference dataset. The CCP4/refmac convention (test set = flag 0, including the historical 0–19 form) is assumed, with the complement taken automatically if flag 0 would be the majority (a phenix-style file where 1 marks free).

10.8 French–Wilson amplitudes

The last step of the merge estimates a Bayesian structure-factor amplitude \(|F|\) for each unique reflection from its intensity \(I\) and error \(\sigma\), so the output carries amplitudes alongside intensities (a naïve \(\sqrt{\max(I,0)}\) turns every weak or negative measurement into a biased — or zero — amplitude). With the Wilson prior for the true intensity \(J\ge 0\) at that resolution,

\( P_\mathrm{acentric}(J) \propto e^{-J/\Sigma},\qquad P_\mathrm{centric}(J) \propto J^{-1/2}\,e^{-J/2\Sigma}, \)

and a Gaussian likelihood \(\mathcal{N}(I;J,\sigma^2)\), the posterior mean amplitude and its uncertainty are

\( \langle |F|\rangle = \frac{\int_0^\infty \sqrt{J}\,\mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J}{\int_0^\infty \mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J},\qquad \sigma_F = \sqrt{\langle J\rangle - \langle|F|\rangle^2}. \)

The prior mean is \(\Sigma = \varepsilon\,\langle I/\varepsilon\rangle_\mathrm{shell}\), where \(\varepsilon\) is the reflection’s epsilon (symmetry-enhancement) multiplicity and \(\langle I/\varepsilon\rangle\) is the Wilson mean in its resolution shell (so reflections on symmetry elements, and each shell, are treated correctly). Strong reflections (\(I>4\sigma\)) short-circuit to \(|F|=\sqrt{I}\), where the French–Wilson bias is negligible; a reflection with an unusable \(I/\sigma\) falls back to \(\sqrt{\max(I,0)}\). The integral is evaluated numerically with a log-shift for stability.

Amplitudes are written as MTZ F/SIGF, mmCIF _refln.F_meas_au/F_meas_sigma_au, and appended to the text HKL, alongside the intensity columns. The same \(|F|\) feed the model-validation step (§14), so the reflection file and the maps use one consistent set of amplitudes.

10.9 Reference data: fixing the space group and resolving the indexing ambiguity

A reference dataset (--reference-mtz) supplies known intensities for the same crystal form, and is used in two ways.

Fix the space group and cell. Unless overridden on the command line (-S for the space group, -C for the cell), the reference’s space group is adopted and its cell is used as the soft reference cell — indexing may still drift the cell within tolerance, so a small mismatch between reference and data is absorbed rather than rejected. This applies to both stills and rotation data.

Resolve the indexing (merohedral) ambiguity. When the lattice symmetry is higher than the crystal’s Laue symmetry (e.g. \(P3\), \(P4\), \(P6\), \(C2\)), more than one indexing of the same lattice is geometrically valid, and the two solutions produce different merged intensities that a self-consistent scale cannot tell apart — only an external reference can. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation \(\mathrm{CC}_\mathrm{ref}\) of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is metric-preserving — only the \(hkl\) labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined; the reference is not used to scale the rotation merge, which stays self-consistent (its \(\mathrm{ISa}\) comes from the data alone). For stills the reference is the per-image scale target of the on-the-fly scaling (§10.2).

10.10 Ice rings at the scale and merge stages

Where the gate of §3.3 has found ice, reflections falling within \(\pm w\) in \(q\) of a hexagonal-ice band (\(w=0.03\) Å\(^{-1}\) offline, about the measured ring half-width) are marked. Marked reflections are excluded where a model is fitted — the per-frame scale \(G\), the per-image correlation, and the \(P1\) merge the space-group search runs on — because ice contamination is a positive bias, not extra scatter, and a least-squares scale absorbs it into \(G\) and into the error-model \(b\), where it damages every other reflection on the same frame. They are kept in the final merge, which is also what the established scaling programs do by default, so the affected shells keep their completeness.

Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge’s own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged \(\mathrm{CC}_{1/2}\) and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by \(-0.001\pm0.018\,\sigma\) — about 2 % of the site height — while removing 1149 unique reflections whose mean \(I/\sigma\) was 3.62 against the dataset’s own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell.

11. Mosaicity and “profile radius” monitoring

11.1 Profile radius (intrinsic excitation-error width)

The “profile radius” is the intrinsic angular width of a reflection — crystal mosaicity plus beam divergence — estimated from the spread of \(\Delta_\mathrm{Ewald}\) over indexed spots, \( R \approx \sqrt{\tfrac{1}{N}\sum_i \Delta_{\mathrm{Ewald},i}^2}. \) When the beam has a finite energy bandwidth, that bandwidth smears each reflection radially by \(\sigma_\mathrm{bw}\approx \mathrm{bandwidth}\cdot\lambda/2d^2\) (largest at high resolution), which also broadens the measured \(\Delta_\mathrm{Ewald}\) spread. Since prediction re-applies the bandwidth term per reflection (§8.2), this contribution is deconvolved from the estimate — \(R^2 = \langle\Delta_\mathrm{Ewald}^2\rangle - \langle\sigma_\mathrm{bw}^2\rangle\) — so that \(R\) is the intrinsic width and bandwidth is not double-counted. Still predictions use an excitation-error cutoff proportional to \(R\).

11.2 Mosaicity from rotation data

For rotation data the mosaicity \(\sigma_M\) is estimated by maximum likelihood from the rocking offsets \(\tau\) of indexed spots, using the XDS reflection-fraction model \(R(\tau;\sigma_M/\zeta)\) (Kabsch 2010): each spot’s exact Bragg angle is located near its frame, \(\zeta\) (the rotation-axis Lorentz component) is computed, and \(\sigma_M\) is chosen to maximize \(\sum_i \log R(\tau_i;\sigma_M/\zeta_i)\).

The \(\phi\) search window for the Bragg angle is set wider than the oscillation, so that reflections recorded at large rocking offset are included. These tail reflections carry most of the information about the mosaic width; a window limited to the oscillation range would truncate the \(\tau\) distribution and bias \(\sigma_M\) low.

The fit uses only the strongest 250 spots of an image, whatever the indexing spot budget (--max-spots) is. A spot is detected when \(I_\mathrm{full}R(\tau)\) clears the finder threshold, so selecting spots by intensity censors on \(R(\tau)\): a deeper list holds proportionally more large-\(\tau\) partially recorded spots and the fit widens with it. Left uncapped, \(\sigma_M\) therefore tracks the spot budget rather than the crystal — and since an over-wide mosaicity mis-states every partiality, the merge degrades sharply with it.

The estimated mosaicity feeds the rotation prediction (how many frames each reflection spans, §8.3) and the rotation partiality (§10.2). It is held fixed during scaling: in the per-image scale fit the mosaicity is degenerate with the scale \(G\) (both rescale the predicted intensity), so refining it there is unstable. A correct mosaicity matters because it controls both how much of each rocking curve is captured and the partiality used to form fulls (§10.6); too small a value truncates the captured curve and over-peaks the partiality, degrading the combined fulls.

12. Auxiliary statistics: ⟨I/σ(I)⟩ and Wilson plot

12.1 Per-shell ⟨I/σ(I)⟩

For monitoring integration quality, Jungfraujoch reports mean \(\langle I/\sigma(I)\rangle\) in a fixed number of resolution shells. Shelling is performed in \(1/d^2\) space (typical of crystallographic practice).

12.2 Wilson plot (B-factor proxy)

A Wilson-type analysis is computed by binning intensities by resolution and fitting: \( \langle I\rangle \propto \exp\!\left(-\frac{B}{2}\frac{1}{d^2}\right), \) i.e. \( \log \langle I\rangle = \mathrm{const} - \frac{B}{2}\left(\frac{1}{d^2}\right). \) A linear regression of \(\log\langle I\rangle\) vs \(1/d^2\) provides an estimate of \(B\), subject to basic quality checks (e.g. \(R^2\) threshold).

A dataset-wide Wilson \(B\) is also estimated over the merged reflections — restricted to the meaningful resolution range (skipping the low-resolution non-linear region below ~4 Å and shells past the signal limit \(\langle I/\sigma\rangle < 1\), so it is insensitive to how far the merged data extend) — and written to the merged mmCIF as _reflns.B_iso_Wilson_estimate, the analogue of XDS’s Wilson-line \(B\). It is diagnostic only and is not fed back into scaling. The per-image estimate (used for the live radiation-damage plot) is accepted only when the fit is well-correlated and physically plausible (\(0 < B < 200\) Ų); on a bad frame (an indexing glitch, too few reflections) the Wilson line runs wildly steep, so an implausible \(B\) is reported as NaN rather than a spurious hundreds-of-Ų value.

13. Space-group determination and merge-level decisions

13.1 Space-group determination

When no space group is supplied, a POINTLESS-like search scores Laue-group symmetry (CC of \(I(h)\) vs \(I(Rh)\) plus merge self-consistency) and detects screw/centering absences from the \(P1\)-merged intensities. Three tests gate a promotion to higher symmetry, all aimed at the merohedral twin, whose twin law forces non-equivalent reflections together and so mimics symmetry:

  1. Merge self-consistency (\(\chi^2\) under the candidate group, relative to the confirmed subgroup). On its own this is not sufficient: it is a ratio to an error model that moves with the amount of data — the parent’s systematic term grows as \(\sigma\) shrinks with \(1/\sqrt{N}\), while a twin’s is already saturated — so its verdict depends on how much data the search saw.

  2. Error-model \(b\) (the intensity-proportional systematic). A genuine symmetry step gains multiplicity without inflating \(b\); merging a twin law’s extra operator inflates it. A \(\chi^2\)-passing promotion is vetoed when \(b\) rises past a bound relative to the confirmed subgroup.

  3. Operator disagreement, a sigma-free statistic \(H=\mathrm{median}\,|I_1-I_2|/(I_1+I_2)\), formed as the ratio of the operators a promotion adds to the parent’s own, measured on the same reflections. Normalising against the parent divides out the systematic floor that symmetry mates carry on real data, which varies by crystal and by operator; a median is used because a twin perturbs every pair whereas a badly-measured minority perturbs only the tail. Where a candidate has several parents of the same order, it is judged against the worst of them, since a rival subgroup can itself contain the twin laws.

The operator correlations are taken on reflections above an \(I/\sigma\) cut, and that cut is capped at the merge’s own \(I/\sigma\) quantile rather than applied as a fixed number. On a search merge whose ISa is below 3, a fixed cut of 3 selects nothing at all, leaving every operator correlation undefined and collapsing the point group to 1. The cap keeps at least the strongest quarter and is inert — the cut stays exactly 3.0 — on a healthy merge.

Several space groups may share an absence pattern exactly. Where they do, the search scores them identically and all of them are named in the result rather than one being reported as the answer: some are enantiomorph pairs, which merged intensities cannot distinguish in principle, and others differ only by a screw condition that the centering condition already implies, so the screw has no observable signature at all. The representative reported first is the lowest space-group number, which is a convention and not a measurement.

The Lorentz factor \(\zeta\) (§8.3) governs how well a reflection can be measured, so when the spindle lies in a plane of the lattice, an operator permuting the two in-plane axes samples a different mixture of measurement qualities than one that only flips signs. The search is therefore run a second time on a merge of only the well-measured observations (--search-min-zeta, rotation default 0.85), both answers are reported, and where they disagree the merge of all the observations decides. The filter discards 40–80 % of the observations, which can starve an operator correlation the full merge confirms and can equally leave an operator confirmed that the full merge refuses, so the decision — the point group as well as the absences, which live in the weak reflections the filter removes — rests on the arm with every observation behind it. A tie (same order, different symmetry) is reported with both candidates named, for trying in molecular replacement.

Centering is accepted when the systematically-absent class is weak relative to the present one by either of two floor-independent tests: its mean signed \(I/\sigma\) well below the present mean, or its rate of individually-significant reflections well below the present class’s own significant rate. The second test covers weak and low-energy data, where a positive intensity floor (background and profile leakage) lifts the absent class’s mean \(I/\sigma\) well above zero and, when the present class is itself weak, carries the plain mean ratio past its bound; a false centering fails both tests, its absent class being as strong as the present one. When several centerings pass, they are ranked by their net systematic absences (absent minus violating), not the gross absent count, so a super-centering (e.g. \(F\) over a true \(C\)) whose extra, only-half-populated absent class dilutes the strength ratio does not out-rank the correct lower centering.

13.2 Twinning check

A Padilla–Yeates \(L\)-test (\(\langle|L|\rangle\), \(\langle L^2\rangle\)) and the second moment \(\langle I^2\rangle/\langle I\rangle^2\) (taken per resolution shell with noise-only shells skipped and Wilson outliers rejected, so a single strong reflection in a collapsed-mean shell cannot skew it) are written to the merged mmCIF as a twinning diagnostic. Twinning is only flagged in Laue classes where a merohedral twin law can exist; the holohedral high-symmetry classes (\(4/mmm\), \(6/mmm\), \(m\bar{3}m\), and \(\bar{3}m\) on a rhombohedral lattice) are exempt, so a low \(\langle|L|\rangle\) there is reported as a statistical artefact rather than twinning.

13.3 Outlier rejection

Merging applies an optional per-observation median-based \(N\sigma\) cut (--reject-outliers, default 6σ for rot3d, off otherwise). The same \(N\sigma\) cut is fed back into the error model: after an initial \(a,b\) fit the parameters are re-fit once on the reflections that survive rejection (dropping any whose squared deviation exceeds \(N\sigma^2\,[a\,\sigma^2 + (b\,\langle I\rangle)^2]\)), so the calibrated errors describe the reflections that actually enter the merge rather than the pre-rejection pool.

13.4 Automatic resolution cutoff

By default the reported/written high-resolution limit is trimmed where \(\mathrm{CC}_{1/2}\) falls off: a logistic is fitted to \(\mathrm{CC}_{1/2}(s)\), and the limit is set one reported-shell width past the point where the fit crosses 0.30 — deliberately “one shell too far”, so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. --scaling-high-resolution overrides the limit and --resolution-cutoff off disables it.

13.5 Practical notes and limitations

  • Bragg integration is profile-fitted by default (per-shell Gaussian profile, Kabsch extraction; §9.3), with plain box summation available as a fallback (--integrator boxsum). The profiles are built per frame from that frame’s strong spots, which suits fast-feedback and serial/streaming use; a profile shared across many frames (as in full offline workflows) is not currently formed.

  • Space-group symmetry beyond centering absences is not enforced during prediction/integration unless the space group is supplied and used downstream.

  • Resolution masking is controllable, and so is every stage of ice-ring handling (§3.3, §10.10). None of it runs unless the crystal is measured to have ice, because the fixed bands are a fixed cost in unique reflections whether it does or not.

  • Rotation vs still modes differ substantially in prediction and scaling: partiality is angle-driven in rotation data, while stills are predicted within an excitation-error window and get their partiality from the default-on per-crystal tilt post-refinement (§10.2) — or unit partiality with --simple-stills.

  • Amplitudes and intensities. The merged output carries both intensities (mmCIF intensity_meas, MTZ IMEAN/SIGIMEAN) and French–Wilson amplitudes (mmCIF F_meas_au, MTZ F/SIGF; §10.8), so a downstream program can refine against either.

14. Model-based validation: R-free against a model and electron-density maps

Offline (rugnux --model model.pdb) the merged data can be scored against a supplied atomic model and initial electron-density maps computed — enough to confirm that a model fits the data and to inspect the density, not a substitute for refinement. The structure itself is not refined; the model is only re-fractionalized into the data unit cell (a rigid cell adjustment, so a deposited model with a slightly different cell still lines up), and the observed amplitudes are the French–Wilson \(|F|\) from §10.8, so the R-free and the maps use exactly the same amplitudes as the written reflection file. The model, structure-factor, bulk-solvent and FFT machinery is provided by GEMMI.

14.1 Model structure factors

The model electron density is sampled on a grid (IT92 X-ray form factors, with a Refmac-compatible Gaussian blur chosen for the grid spacing) and Fourier-transformed to structure factors \(F_\mathrm{calc}(hkl)\) up to the data resolution.

14.2 Bulk solvent and scaling

A flat bulk-solvent mask around the model is transformed to \(F_\mathrm{mask}\), and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale \(k\), an anisotropic \(B\), and the flat-solvent parameters \(k_\mathrm{sol}, B_\mathrm{sol}\):

\( F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2. \)

This is the standard, few-parameter scaling model used by refinement programs. No free-form per-resolution-shell rescale is applied: such a rescale is dataset-specific and reshapes each map’s radial amplitude profile differently, which would make maps from a multi-dataset campaign no longer directly comparable.

14.3 R-work and R-free

Crystallographic R-factors are reported over the work and free sets (the §10.7 flags):

\( R = \frac{\sum \big|\,|F_o| - |F_\mathrm{model}|\,\big|}{\sum |F_o|}, \)

with R-free the same sum restricted to the free set. Note that the scaling of §14.2 is fitted over all reflections, work and free alike — its few parameters (\(k\), an anisotropic \(B\), \(k_\mathrm{sol}\), \(B_\mathrm{sol}\)) are far too few to absorb individual reflections, but R-free here is strictly “free of refinement”, not free of the scaling fit.

14.4 Electron-density maps

Two maps are formed with the model phases \(\varphi_\mathrm{model}\): a \(2F_o-F_c\) map, coefficients \((2|F_o|-|F_\mathrm{model}|)\,e^{i\varphi_\mathrm{model}}\), and an \(F_o-F_c\) difference map, \((|F_o|-|F_\mathrm{model}|)\,e^{i\varphi_\mathrm{model}}\), each inverse-Fourier-transformed to a real-space CCP4 map (<prefix>_2fofc.ccp4, <prefix>_fofc.ccp4). A map-coefficient MTZ (<prefix>_maps.mtz: FP, FC, PHIC, FWT/PHWT, DELFWT/PHDELWT, FREE) is written alongside so the maps can be reopened or rebuilt in Coot / PyMOL. These are unweighted difference coefficients (no \(\sigma_A\) / figure-of-merit weighting), which is why they are described as initial maps.

14.5 Aligning the data to the model: enantiomorph and indexing ambiguity

The model fixes a definite hand and indexing, but the merged data need not share them, so before comparison the observed reflections are brought into the model’s frame.

  • Enantiomorph / screw. When the data space group is the enantiomorph of the model’s (e.g. data \(P4_12_12\), model \(P4_32_12\); or \(P3_1/P3_2\)), the two are indistinguishable from merged intensities — \(|F_\mathrm{calc}|\) is invariant under the change of hand, so R-free cannot choose between them and probing would be meaningless. The hand is therefore taken from the model: the observed reflections are reindexed by the change-of-hand operator into the model’s enantiomorph. Only the map phases (the density’s hand) depend on this choice.

  • Indexing (merohedral) ambiguity. When the crystal has a merohedral ambiguity (§10.9), the observed intensities do differ between indexings, and the right one is chosen against the best available reference. If a reference MTZ was supplied, the data were already reindexed to agree with it (§10.9 — by the reference-intensity correlation, at the merge stage for rotation data or per image in stills scaling), and model validation keeps that authoritative choice. Only with a model and no reference does validation resolve the ambiguity itself, as a fallback: the scaled model is fit to each reindexing of the data (identity plus the twin-law cosets) and the one giving the lowest R-free is kept. This matters for a multi-dataset campaign — a single shared reference fixes one indexing convention for every dataset, whereas an independent per-dataset lowest-R-free choice could send borderline datasets to different conventions. A no-op either way for a holohedral crystal (no twin laws).

\ No newline at end of file diff --git a/DEPLOYMENT.html b/DEPLOYMENT.html new file mode 100644 index 00000000..fe30c4f4 --- /dev/null +++ b/DEPLOYMENT.html @@ -0,0 +1,38 @@ + Deployment — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Deployment

To deploy Jungfraujoch, one needs to follow four steps:

  1. Install main Jungfraujoch code and frontend web interface

  2. Flash the U55C FPGA card with a proper image and install Linux kernel driver

  3. Install Jungfraujoch writer

  4. Install Python OpenAPI client

Installation procedure depend a lot on the operating system. For RedHat Enterprise Linux 8/9, Rocky 8/9, Ubuntu 22.04/24.04 or compatible, installation can be done with prebuilt packages from the package repositories and is relatively straightforward. For other systems one needs to build software from source. Both ways will be presented. What each released package contains, and what it needs on the machine, is described in Release contents.

Install main Jungfraujoch code and frontend web interface

On RHEL 8 systems there is a jfjoch-<version>-1.el8.x86_64.rpm that needs to be installed and contains all the necessary software and web interface.

On other OSes one needs to compile Jungfraujoch from source (from the repo directory):

$ mkdir build
+$ cd build
+$ cmake .. -DCMAKE_INSTALL_PREFIX=<directory to install>
+$ make
+$ sudo make install  
+

For manual installation, we recommend to use non-standard directory (like /opt/jfjoch), to facilitate upgrades and removal. For DKMS to manage kernel module sources it is necessary to copy driver sources to /usr/src/jfjoch-<VERSION> directory. This requires extra flag in cmake -DJFJOCH_INSTALL_DRIVER_SOURCE=ON.

Frontend web user interface has to be built separately with:

$ cd build
+$ make frontend
+

Frontend files (.html and .js) will be placed in frontend/dist (outside of build/ directory!) and has to be copied to a general location, e.g. /usr/local/jfjoch/frontend or /opt/jfjoch/frotend.

Flash the U55C FPGA card with a proper image and install Linux kernel driver.

Firmware flashing

  1. Check that the card is detected by OS with “lspci |grep Xilinx” and check the PCIe bus/device/function (BDF) number, 11:00.0 in this case:

$ lspci |grep Xilinx
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2)
+

Note the device number 3450 that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and rev 2 identifying release of the firmware.

  1. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given):

$ sudo lspci -vv -s <PCIe slot number>
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450
+(...)
+LnkSta:     Speed 16GT/s (ok), Width x8 (ok)
+(...)
+

  1. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn’t allways reach correct timing).

  2. Flash the card with xbflash.qspi tool (part of Jungfraujoch). For fresh card use:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above> --bar-offset 0x1f06000 
+

For card that was already flashed with Jungfraujoch images:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above>
+

It is necessary to confirm the operation by pressing Y key or one can add --force option to avoid confirmation. It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions.

  1. Cold reboot:

sudo ipmitool chassis power cycle
+

Install PCIe driver

For first run it is though recommended to try the driver without installing to the kernel directory:

$ cd fpga/pcie_driver
+$ make
+$ sudo insmod jfjoch.ko
+

Check with dmesg that the device was properly found:

$ dmesg |grep jfjoch
+[  431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142)
+[  431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a
+

If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. On RHEL 8 you can install the prebuilt jfjoch-driver-dkms package from the Gitea package registry. On other systems follow procedure in PCIe driver.

NOTE: Driver installation procedure on non-RHEL 8 systems is not well understood/optimized at the moment.

NOTE: In case driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk if driver is updated:

$ sudo dracut -f
+

Configure network

Configure switch according to FPGA network guide - specifically set manual speed and turn off auto-negotiation for the port used to connect U55C card and connect card to switch.

Running Jungfraujoch software

Main Jungfraujoch service is called jfjoch_broker. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. It is recommended to run the service as systemd service.

jfjoch_broker takes two parameters: JSON configuration file and HTTP port (default is 5232). Example JSON files are placed in etc/ folder. JSON file format is also explained in the OpenAPI definition, as jfjoch_settings data structure.

When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring.

Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one jfjoch_broker on a single machine, each confined to a disjoint subset of GPUs, set CUDA_VISIBLE_DEVICES; setting CUDA_DEVICE_ORDER=PCI_BUS_ID keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host:

CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232
+CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233
+

To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. These need to be obtained from the PSI Detector Group.

Card verification

To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use jfjoch_fpga_test tool. For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs:

jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000
+

Or 1M pixel system with one FPGA card:

jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000
+

Install Jungfraujoch writer

Jungfraujoch writer is an additional service, that can connect to jfjoch_broker ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard.

At the moment it is better to have a separate machine, with access to distributed file system, for writing images.

Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands:

mkdir build
+cd build
+cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch
+

Install Jungfraujoch image viewer

Jungfraujoch viewer is X-ray diffraction image viewer, that is optimized to open Jungfraujoch HDF5 files.

The viewer is a Qt application and it requires recent version of the library, therefore it is an optional dependency.

To include it in the building of Jungfraujoch use -DJFJOCH_VIEWER_BUILD=ON directive for CMake:

mkdir build
+cd build
+cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch
+

Install Jungfraujoch Python client

Use pip:

pip install jfjoch-client
+
\ No newline at end of file diff --git a/DETECTORS.html b/DETECTORS.html new file mode 100644 index 00000000..1c86f0ab --- /dev/null +++ b/DETECTORS.html @@ -0,0 +1 @@ + Supported detectors — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Supported detectors

PSI detectors

Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfruajoch controls the detector via statically compiled slsDetectorPackage into its source code. It is important that detector firmware has to match slsDetectorPackage version used in Jungfraujoch. The default is 8.0.2; 9.2.0 is built with the SLS9=ON CMake option and published in the slsdet9 package repositories. See PSI Detector group website for details.

DECTRIS detectors

Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles output data stream of the DCU. This mode, called “lite” mode, doesn’t use FPGA boards, but mostly CPUs and GPUs for indexing. The mode is currently experimental and intended for low data rates (100 Hz).

\ No newline at end of file diff --git a/DETECTOR_GEOMETRY.html b/DETECTOR_GEOMETRY.html new file mode 100644 index 00000000..983a9c31 --- /dev/null +++ b/DETECTOR_GEOMETRY.html @@ -0,0 +1 @@ + Detector geometry — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Detector geometry

At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location vs. detector frame. It is not recommended to place detector modules stacked.

The simplest case is detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance and wavelength.

For more complex case, one can provide tilt of the detector rotation in PyFAI convention. This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector is defined with three rotation angles: rot1 (rotating detector right), rot2 (rotating detector downwards), rot3 (rotating detector clockwise). See PyFAI documentation for more details.

What a pixel coordinate means: (0, 0) is the centre of the first pixel

Pixel coordinates in Jungfraujoch and rugnux are 0-based and pixel-centred: an integer coordinate is the centre of that pixel, so pixel i covers [i − 0.5, i + 0.5) and the sensor spans −0.5 … width − 0.5. A beam centre of 948.0 × 546.0 sits in the middle of pixel [546][948], not on any of its corners; 948.5 is the boundary between pixel 948 and 949.

This holds throughout the code: spot and reflection centroids are intensity-weighted sums of the integer pixel indices, the resolution and azimuthal-bin maps evaluate pixel (col, row) at exactly (col, row), and a fractional coordinate is turned back into a pixel index by rounding, not by truncation. The same convention applies to every coordinate the system exposes — the beam centre (beam_x_pxl/beam_y_pxl in the API and broker configuration, --beam-x/--beam-y in rugnux, beam_center_x/beam_center_y in NXmx and in the CBOR stream), the spot and predicted-reflection positions written to HDF5, and the PONI reported by --mode calibration.

Other programs place the origin differently, and the difference is worth half a pixel — enough to matter when a geometry is copied between programs and then refined:

Convention

Beam centre equivalent to our x = 948.0

Jungfraujoch, rugnux

948.0

XDS (ORGX/ORGY)

949.0 — also pixel-centred, but pixels are numbered from 1

Measured from the edge of the sensor, in length units — pyFAI (Poni1/Poni2), DIALS/dxtbx

(948.0 + 0.5) × pixel size, because the centre of pixel i is at (i + 0.5) × pixel size from the edge

The .poni file written by rugnux --mode calibration is in pyFAI’s frame and so already carries that half pixel; the pixel values the same run reports are ours.

Macromolecular crystallography convention for the vertical direction

One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, with Y values increasing downwards. This is also consistent with computer image formats.

However, other techniques (as well as internal operation of PSI X-ray detectors) might follow convention, for point (0,0) being in the bottom left corner and Y values increasing upwards. Such a convention is used, for example, by PyFAI.

In general, convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows mirroring detector in Y.

Extra care has to be taken by the user to ensure that no errors are made.

\ No newline at end of file diff --git a/FPGA.html b/FPGA.html new file mode 100644 index 00000000..a6947685 --- /dev/null +++ b/FPGA.html @@ -0,0 +1,16 @@ + FPGA smartNIC — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA smartNIC

See separate document for installation instructions.

Hardware

Currently supported FPGA is only Xilinx Alveo U55C.

See AMD/Xilinx webpage for card user guide (UG1469). According to the user guide:

Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling.
+

Card needs to be placed in PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C.

Two variants of the card are available:

  • 100g - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch.

  • 8x10g - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

See network documentation for details of network.

Building firmware

The firmware build targets are generated by CMake only when vivado and vitis_hls are detected in the path, and the Vivado version has to match the one below precisely.

Xilinx Vivado

The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version 2022.2 installed on the machine. Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - specifically newer versions of Vivado will not work.

In additional to Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary:

  • Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see Xilinx website, to build 100g design.

  • Paid 10G/25G Subsystem for Ultrascale+ to build 8x10g design. PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams generated by PSI continuous integration pipeline for 8x10g is only allowed for non-commercial use.

HLS compilation

Make HLS routines:

mkdir build
+cd build
+cmake ..
+make hls
+

Synthesis

Create PCIe 100g bitstream with the following command:

mkdir build
+cd build
+cmake ..
+make pcie_100g
+

and 8x10g:

mkdir build
+cd build
+cmake ..
+make pcie_8x10g
+

When Vivado is not present

During CMake execution, the following executables: vivado and vitis_hls must be present in the path. If not, build targets will not be generated, and such or similar error message will show up:

$ make pcie_100g
+make: *** No rule to make target 'pcie_100g'.  Stop.
+

Firmware releases

The firmware is stable and is carried from version to version: the MCS files attached to a release are normally the ones from the release before it (see Release contents). When it does need to change, it is rebuilt with the targets above on a machine with Vivado.

Frame generator

Jungfraujoch card is equipped with frame generator. It allows to simulate JUNGFRAU detector without having access to such system. It is placed in parallel to Ethernet MAC - so it is placed before the network stack and before any processing happening on the card. In the future a redirection will be possible to send the simulated stream through the 100G TX network link. Frame generator is written in HLS and controlled with AXI-Lite.

\ No newline at end of file diff --git a/FPGA_DATA_ANALYSIS.html b/FPGA_DATA_ANALYSIS.html new file mode 100644 index 00000000..87fdcce4 --- /dev/null +++ b/FPGA_DATA_ANALYSIS.html @@ -0,0 +1 @@ + FPGA data analysis — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA data analysis

Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities.

Pixel mask

Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) and will be excluded from a subsequent analysis.

ADU histogram

Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature of unconverted values. This is done on a module-basis and works with bins with 32 ADU width.

For EIGER this can be used as just a histogram procedure.

JUNGFRAU conversion

For JUNGFRAU module images are converted from ADUs to energy value and divided by a given number to keV units. Result of the operation is rounded to integers.

Pixel thresholding

Pixel range can be specified. Pixels below a minimum threshold will be assigned zero. Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled.

Frame summation

Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together.

Azimuthal integration

To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. Corrections were implemented according to formulas developed by Jensen et al. (J. Synchr. Rad., 29, 1420-1428, 2022).

Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited as 2048 per detector module. This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit pixed point integer implementation and is tuned for standard SAXS/WAXS range.

As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows.

Spot finding

Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels:

  1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run).

  2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding,

  3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation,

  4. Pixel value - pixels above certain threshold value can be marked as strong,

  5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong,

  6. Connected pixels - strong pixels can be discarded if they are “alone”, so their 8 directly neighboring pixels are not counted as strong pixels.

While besides bad pixels criterion, all the above are optional (can be turned off), only pixels that fulfill all enabled criteria are selected as strong pixels.

SNR ratio calculation

Signal-to-noise ratio is calculated for a rectangular area. In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. This is dictated by the data flow within the FPGA. In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. Given very large box size, approximation are made, for example that N ≈ N-1 in calculating standard deviation.

Region-of-interest (ROI) integration

There are 16 ROIs, and the ROI map holds a 16-bit mask per pixel, so a pixel can belong to any subset of them (including none). For each ROI, sum, sum of squares, max count, and number of valid pixels will be calculated. Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis.

ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal (Q-range with an optional φ-sector) — are also evaluated on CPU by the shared image_analysis/roi/ engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path.

Pixel statisitics

The following statistics are collected for each module:

  • Number of masked pixels

  • Number of saturated pixels (excl.masked)

  • Number of error pixels (excl. masked)

  • Sum of valid pixels in the module

  • Minimum value of valid pixels in the module

  • Maximum value of valid pixels in the module Valid pixels are not masked, not saturated, not error pixels.

Square root compression

Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. Scheme was described in Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020. Pixel value X is replaced with round(sqrt(N*N*X)), i.e. round(N*sqrt(X)), where N is integer constant in range 1 to 16. N is what the host writes to the sqrtmult register; the FPGA squares it before multiplying the pixel value.

\ No newline at end of file diff --git a/FPGA_DESIGN.html b/FPGA_DESIGN.html new file mode 100644 index 00000000..3f73f114 --- /dev/null +++ b/FPGA_DESIGN.html @@ -0,0 +1 @@ + FPGA data flow — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA data flow

The following steps are performed on FPGA (in the order of operation):

  1. UDP header decoding

  2. SLS detector header decoding

  3. State machine that controls data acquisition (start/stop/cancel)

  4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules

  5. ADU histogram for JUNGFRAU

  6. Mask pixels from missing packets with special value

  7. Reorder lines for EIGER to form a proper module

  8. Mask pixels based on provided pixel mask

  9. JUNGFRAU conversion with gain and pedestal corrections

  10. Threshold to zero pixels below certain count value

  11. Integration according to predefined map (e.g., 1D azimuthal integration)

  12. Spot finding

  13. ROI calculation

  14. Image lossy compression using N*sqrt(pixel) values

  15. Send images, analysis results and metadata to host memory via PCI Express

Each step has dedicated core, written in the high-level synthesis. Exact operation of cores for data analysis is explained in dedicated document.

\ No newline at end of file diff --git a/FPGA_LICENSE.html b/FPGA_LICENSE.html new file mode 100644 index 00000000..864fcf80 --- /dev/null +++ b/FPGA_LICENSE.html @@ -0,0 +1,7 @@ + FPGA license — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA license

FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. The license is equivalent of GNU Public License with adaptations for hardware. See OHL webpage for details and FAQs.

CERN Open Hardware Licence Version 2 - Strongly Reciprocal

Preamble

CERN has developed this licence to promote collaboration among hardware designers and to provide a legal tool which supports the freedom to use, study, modify, share and distribute hardware designs and products based on those designs. Version 2 of the CERN Open Hardware Licence comes in three variants: CERN-OHL-P (permissive); and two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this licence, CERN-OHL-S (strongly reciprocal).

The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in unmodified form only.

Use of this Licence does not imply any endorsement by CERN of any Licensor or their designs nor does it imply any involvement by CERN in their development.

1 Definitions

1.1 ‘Licence’ means this CERN-OHL-S.

1.2 ‘Compatible Licence’ means

a) any earlier version of the CERN Open Hardware licence, or

b) any version of the CERN-OHL-S, or

c) any licence which permits You to treat the Source to which it applies as licensed under CERN-OHL-S provided that on Conveyance of any such Source, or any associated Product You treat the Source in question as being licensed under CERN-OHL-S.

1.3 ‘Source’ means information such as design materials or digital code which can be applied to Make or test a Product or to prepare a Product for use, Conveyance or sale, regardless of its medium or how it is expressed. It may include Notices.

1.4 ‘Covered Source’ means Source that is explicitly made available under this Licence.

1.5 ‘Product’ means any device, component, work or physical object, whether in finished or intermediate form, arising from the use, application or processing of Covered Source.

1.6 ‘Make’ means to create or configure something, whether by manufacture, assembly, compiling, loading or applying Covered Source or another Product or otherwise.

1.7 ‘Available Component’ means any part, sub-assembly, library or code which:

a) is licensed to You as Complete Source under a Compatible Licence; or

b) is available, at the time a Product or the Source containing it is first Conveyed, to You and any other prospective licensees

i) as a physical part with sufficient rights and information (including any configuration and programming files and information about its characteristics and interfaces) to enable it either to be Made itself, or to be sourced and used to Make the Product; or ii) as part of the normal distribution of a tool used to design or Make the Product.

1.8 ‘Complete Source’ means the set of all Source necessary to Make a Product, in the preferred form for making modifications, including necessary installation and interfacing information both for the Product, and for any included Available Components. If the format is proprietary, it must also be made available in a format (if the proprietary tool can create it) which is viewable with a tool available to potential licensees and licensed under a licence approved by the Free Software Foundation or the Open Source Initiative. Complete Source need not include the Source of any Available Component, provided that You include in the Complete Source sufficient information to enable a recipient to Make or source and use the Available Component to Make the Product.

1.9 ‘Source Location’ means a location where a Licensor has placed Covered Source, and which that Licensor reasonably believes will remain easily accessible for at least three years for anyone to obtain a digital copy.

1.10 ‘Notice’ means copyright, acknowledgement and trademark notices, Source Location references, modification notices (subsection 3.3(b)) and all notices that refer to this Licence and to the disclaimer of warranties that are included in the Covered Source.

1.11 ‘Licensee’ or ‘You’ means any person exercising rights under this Licence.

1.12 ‘Licensor’ means a natural or legal person who creates or modifies Covered Source. A person may be a Licensee and a Licensor at the same time.

1.13 ‘Convey’ means to communicate to the public or distribute.

2 Applicability

2.1 This Licence governs the use, copying, modification, Conveying of Covered Source and Products, and the Making of Products. By exercising any right granted under this Licence, You irrevocably accept these terms and conditions.

2.2 This Licence is granted by the Licensor directly to You, and shall apply worldwide and without limitation in time.

2.3 You shall not attempt to restrict by contract or otherwise the rights granted under this Licence to other Licensees.

2.4 This Licence is not intended to restrict fair use, fair dealing, or any other similar right.

3 Copying, Modifying and Conveying Covered Source

3.1 You may copy and Convey verbatim copies of Covered Source, in any medium, provided You retain all Notices.

3.2 You may modify Covered Source, other than Notices, provided that You irrevocably undertake to make that modified Covered Source available from a Source Location should You Convey a Product in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source. In each case subsection 3.3 shall apply.

  You may only delete Notices if they are no longer applicable to
+  the corresponding Covered Source as modified by You and You may
+  add additional Notices applicable to Your modifications.
+  Including Covered Source in a larger work is modifying the
+  Covered Source, and the larger work becomes modified Covered
+  Source.
+

3.3 You may Convey modified Covered Source (with the effect that You shall also become a Licensor) provided that You:

a) retain Notices as required in subsection 3.2;

b) add a Notice to the modified Covered Source stating that You have modified it, with the date and brief description of how You have modified it;

c) add a Source Location Notice for the modified Covered Source if You Convey in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source; and

d) license the modified Covered Source under the terms and conditions of this Licence (or, as set out in subsection 8.3, a later version, if permitted by the licence of the original Covered Source). Such modified Covered Source must be licensed as a whole, but excluding Available Components contained in it, which remain licensed under their own applicable licences.

4 Making and Conveying Products

You may Make Products, and/or Convey them, provided that You either provide each recipient with a copy of the Complete Source or ensure that each recipient is notified of the Source Location of the Complete Source. That Complete Source is Covered Source, and You must accordingly satisfy Your obligations set out in subsection 3.3. If specified in a Notice, the Product must visibly and securely display the Source Location on it or its packaging or documentation in the manner specified in that Notice.

5 Research and Development

You may Convey Covered Source, modified Covered Source or Products to a legal entity carrying out development, testing or quality assurance work on Your behalf provided that the work is performed on terms which prevent the entity from both using the Source or Products for its own internal purposes and Conveying the Source or Products or any modifications to them to any person other than You. Any modifications made by the entity shall be deemed to be made by You pursuant to subsection 3.2.

6 DISCLAIMER AND LIABILITY

6.1 DISCLAIMER OF WARRANTY – The Covered Source and any Products are provided ‘as is’ and any express or implied warranties, including, but not limited to, implied warranties of merchantability, of satisfactory quality, non-infringement of third party rights, and fitness for a particular purpose or use are disclaimed in respect of any Source or Product to the maximum extent permitted by law. The Licensor makes no representation that any Source or Product does not or will not infringe any patent, copyright, trade secret or other proprietary right. The entire risk as to the use, quality, and performance of any Source or Product shall be with You and not the Licensor. This disclaimer of warranty is an essential part of this Licence and a condition for the grant of any rights granted under this Licence.

6.2 EXCLUSION AND LIMITATION OF LIABILITY – The Licensor shall, to the maximum extent permitted by law, have no liability for direct, indirect, special, incidental, consequential, exemplary, punitive or other damages of any character including, without limitation, procurement of substitute goods or services, loss of use, data or profits, or business interruption, however caused and on any theory of contract, warranty, tort (including negligence), product liability or otherwise, arising in any way in relation to the Covered Source, modified Covered Source and/or the Making or Conveyance of a Product, even if advised of the possibility of such damages, and You shall hold the Licensor(s) free and harmless from any liability, costs, damages, fees and expenses, including claims by third parties, in relation to such use.

7 Patents

7.1 Subject to the terms and conditions of this Licence, each Licensor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in subsections 7.2 and 8.4) patent licence to Make, have Made, use, offer to sell, sell, import, and otherwise transfer the Covered Source and Products, where such licence applies only to those patent claims licensable by such Licensor that are necessarily infringed by exercising rights under the Covered Source as Conveyed by that Licensor.

7.2 If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Covered Source or a Product constitutes direct or contributory patent infringement, or You seek any declaration that a patent licensed to You under this Licence is invalid or unenforceable then any rights granted to You under this Licence shall terminate as of the date such process is initiated.

8 General

8.1 If any provisions of this Licence are or subsequently become invalid or unenforceable for any reason, the remaining provisions shall remain effective.

8.2 You shall not use any of the name (including acronyms and abbreviations), image, or logo by which the Licensor or CERN is known, except where needed to comply with section 3, or where the use is otherwise allowed by law. Any such permitted use shall be factual and shall not be made so as to suggest any kind of endorsement or implication of involvement by the Licensor or its personnel.

8.3 CERN may publish updated versions and variants of this Licence which it considers to be in the spirit of this version, but may differ in detail to address new problems or concerns. New versions will be published with a unique version number and a variant identifier specifying the variant. If the Licensor has specified that a given variant applies to the Covered Source without specifying a version, You may treat that Covered Source as being released under any version of the CERN-OHL with that variant. If no variant is specified, the Covered Source shall be treated as being released under CERN-OHL-S. The Licensor may also specify that the Covered Source is subject to a specific version of the CERN-OHL or any later version in which case You may apply this or any later version of CERN-OHL with the same variant identifier published by CERN.

8.4 This Licence shall terminate with immediate effect if You fail to comply with any of its terms and conditions.

8.5 However, if You cease all breaches of this Licence, then Your Licence from any Licensor is reinstated unless such Licensor has terminated this Licence by giving You, while You remain in breach, a notice specifying the breach and requiring You to cure it within 30 days, and You have failed to come into compliance in all material respects by the end of the 30 day period. Should You repeat the breach after receipt of a cure notice and subsequent reinstatement, this Licence will terminate immediately and permanently. Section 6 shall continue to apply after any termination.

8.6 This Licence shall not be enforceable except by a Licensor acting as such, and third party beneficiary rights are specifically excluded.

\ No newline at end of file diff --git a/FPGA_NETWORK.html b/FPGA_NETWORK.html new file mode 100644 index 00000000..47282780 --- /dev/null +++ b/FPGA_NETWORK.html @@ -0,0 +1 @@ + FPGA network — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA network

The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 is lower port (when PCIe connector is on the bottom). The card FPGA design is offered in two variants 100g and 8x10g. These have different behavior regarding the network:

100g this variant operates QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a switch. QSFP28 transceivers are necessary.

8x10g this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

Transceivers

AMD doesn’t provide transceiver compatibility matrix for Alveo U55C. In our experience operating the card we haven’t seen issues with transceivers from various providers (FS.com, Mellanox, Finnisar). We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn’t support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven’t encountered such a situation.

Switch configuration

Special care has to be taken for switch operation, given the FPGA core doesn’t support auto-negotiation. It is necessary to configure switch port to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000).

Network LEDs

Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs means all ports connected, orange LEDs at least one of the ports connected.

Network stack

Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed.

The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. The card won’t respond to broadcast ICMP pings.

Each link can be put in direct mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. This settings should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any 10 Gbit/s link on the same card. Currently direct mode is turned OFF for 100g design and ON for 8x10g design. This can be manually adjusted for each link.

\ No newline at end of file diff --git a/FPGA_PCIE_DRIVER.html b/FPGA_PCIE_DRIVER.html new file mode 100644 index 00000000..abf90b09 --- /dev/null +++ b/FPGA_PCIE_DRIVER.html @@ -0,0 +1,10 @@ + FPGA PCIe driver — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA PCIe driver

Compilation

To compile kernel module type:

make
+

Installation

To install kernel module, you need to have root permissions and run:

sudo make install
+

Loading driver into kernel

After installing the kernel driver, it should be possible to insert it into the kernel via:

modprobe jfjoch
+

Ownership of the character devices

By default, character devices /dev/jfjoch<device number> are owned by root (user/group) and are not accessible by others. This means that jfjoch_broker must be running as superuser, which might not be optimal for security reasons in most cases. The behavior can be changed by creating udev rules. Create a file called /etc/udev/rules.d/99-jfjoch.rules with the following content:

KERNEL=="jfjoch*" OWNER="<UNIX username>" GROUP="<UNIX group>"
+

It is OK to provide only group, for example to make the devices accessible by group jungfrau:

KERNEL=="jfjoch*" GROUP="jungfrau"
+

DKMS

To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. For other systems, it is necessary to follow the procedure below, though it is not well tested.

This first requires to install DKMS - for RHEL it is available via EPEL repository:

sudo dnf install dkms
+

Then use script provided in the driver directory to copy driver code to DKMS directory:

./install_dkms.sh
+

If upgrading the driver, please first remove current driver from DKMS system:

dkms remove jfjoch -v <version> --all
+

Driver parameters

Currently, there is one driver parameter nbuffers, that defines count of exchange buffers (see below). This can be adjusted in the modprobe operation, for example:

modprobe jfjoch nbuffers=1024
+

Exchange buffers

The parameter defines number of buffers used to exchange data between card and host application. Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. These buffers are used by both card-to-host and host-to-card operations.

Buffers use special allocation, as they are continuous in physical address space, which helps the FPGA card to transfer all data associated with detector module in two DMA transfers (one data, one metadata). Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in power of two, 4 MiB is safe number for one buffer size. Buffer can be mapped into user space, but performing mmap system call on the /dev/jfjoch<number of device> character device.

Buffer count can be adjusted by setting nbuffers parameter. There are two considerations for setting optimal value:

  1. For card-to-host transfers, minimal value is roughly <number of threads in receiver> * <number of modules processed by thread; usually equal to number of modules per card>, this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64.

  2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. So minimal count is <number of modules> * (3 + 3 * <number of storage cells>).

Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration.

Known problems

To avoid inconsistent behavior, this driver won’t load if release number differs between the kernel driver and FPGA card.

CMake file

While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. It is not made for actual compilation of the kernel driver and should not be used for that purpose.

Character device access

For each FPGA device a character device is created called /dev/jfjoch<number of device>. When device is opened two operations are possible: mmap() to map exchange buffers ioctl() to communicate with the cards Interfacing should be done through the JungfraujochDevice class in fpga/host_library directory.

Sysfs access

Certain performance counters can be read through sysfs mechanism in the kernel. One needs to cat files in /sys/class/misc/jfjoch<number of device>/ directory.

RHEL 9.5+ issue

RedHat Enterprise Linux 9.5 backported modification to settings virtual memory flags from Linux kernel 6.3, while still operating kernel version 5.14. It is complicated to come up with a single rule to select when newer functions should be used, so it works with RHEL 9.5+, while still being compatible with other Linux distributions. It is even more complex given not all RHEL compatible distributions adopted the change at the same version. For the moment the quick fix is to define an environment variable HAVE_VM_FLAGS_SET before making the kernel.

\ No newline at end of file diff --git a/FPGA_SETTINGS.html b/FPGA_SETTINGS.html new file mode 100644 index 00000000..a919a4aa --- /dev/null +++ b/FPGA_SETTINGS.html @@ -0,0 +1 @@ + FPGA advanced reference — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

FPGA advanced reference

Register map

FPGA setup can be done via registers:

Address

Bits

Meaning

Mode

Notes

0x000000 - 0x00FFFF

Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory)

0x010000

32

Action Control Register

Bit 0 - Action start

R/W

Bit 1 - Action idle

R

Bit 2 - Action cancel

R/W

cleared on reset or action start

Bit 3 - Clear network counters

R/W

cleared on reset

Bit 12:4 - Debug signals (see action_config.v for details)

R

Bit 16 - AXI Mailbox interrupt 0

R

0x010004

32

Reserved

-

0x010008

32

Reserved

-

0x01000C

32

GIT SHA1

R

0x010010

32

Reserved

R

0x010014

32

Reserved

R

0x010018

32

Jungfraujoch FPGA variant

R

0x01001C

32

Reserved

R

0x010020

32

Max. number supported detector modules

R

constant

0x010024

32

Reserved

R

constant

0x010028

64

Pipeline stalls before writing to host memory

R

reset on action start

0x010030

64

Pipeline stalls before accessing HBM

R

reset on action start

0x010038

32

FIFO status (see action_config.v for details)

R

0x01003C

32

Size of single HBM channel in bytes (default value for the particular card)

R/W

should not be altered for standard operation

0x010040

64

Packets processed by the action

R

cleared on reset or action start

0x010048

64

Valid ethernet packets

R

cleared on reset

0x010050

64

Valid ICMP packets

R

cleared on reset

0x010058

64

Valid UDP packets

R

cleared on reset

0x010060

64

Valid detector packets processed by the card

R

cleared on reset

0x010068

64

Packets flagged as errors by CMAC

R

cleared on reset

0x010070

64

Pipeline stalls before data processing

R

reset on action start

0x010078

64

AXI-beats before accessing HBM

R

reset on action start

0x010080

64

AXI-beats before data processing

R

reset on action start

0x010088

64

AXI-beats before host writer

R

reset on action start

0x010090

64

Last encountered SwissFEL pulse ID

R

cleared on reset

0x010100

32

Spot finder photon count threshold

R/W

0x010104

32

Spot finder signal-to-noise ratio threshold (single-precision float)

R/W

0x010200

64

MAC address source for internal frame generator

R/W

network byte order

0x010208

32

IPv4 address source for internal frame generator

R/W

network byte order

0x01020C

32

Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.)

R/W

0x010210

32

Data collection mode

R/W

Bit 0 - Conversion to photons

Bit 1 - Output extend to 32-bit

Bit 2 - Output is unsigned integer

Bit 3 - Use sq. root lossy compression

Bit 7 - JUNGFRAU fixed G1 mode

Bit 8 - Set to zero values below threshold

Bit 16:31 - Data collection ID (carried with completions)

0x010214

32

Photon energy in keV (single-precision float)

R/W

0x010218

32

Number of frames expected in the data collection (defines termination condition)

R/W

0x01021C

32

Number of storage cells

R/W

0x010220

32

Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.)

R/W

0x010224

32

Coefficient for sq. root compression (need to set bit in data collection mode to apply)

R/W

0x010225

32

Threshold; set values below set to zero (need to set bit in data collection mode to apply)

R/W

0x030000 - 0x03FFFF

AXI Mailbox for Work Request / Work Completion

See Xilinx PG114 for register map

0x040000 - 0x04FFFF

QuadSPI flash

See Xilinx PG153 for register map

0x050000 - 0x05FFFF

Interrupt controller

See Xilinx PG099 for register map

0x060000 - 0x06FFFF

Load calibration (HLS)

0x070000 - 0x07FFFF

AXI Firewall

See Xilinx PG293 for register map

0x080000 - 0x08FFFF

Frame generator (HLS)

0x090000 - 0x09FFFF

PCIe DMA control

See Xilinx PG195 for register map

0x0A0000 - 0x0AFFFF

I2C clock generator

See Xilinx PG195 for register map

0x0C0000 - 0x0FFFFF

Xilinx Card Management Solution Subsystem management subsystem

See Xilinx PG348 for register map

0x100000 - 0x10FFFF

MAC 10G / CMAC 100G

See Xilinx PG210/PG203 for register map

0x110000 - 0x11FFFF

MAC 10G

See Xilinx PG210 for register map

0x120000 - 0x12FFFF

MAC 10G

See Xilinx PG210 for register map

0x130000 - 0x13FFFF

MAC 10G

See Xilinx PG210 for register map

0x140000 - 0x14FFFF

MAC 10G

See Xilinx PG210 for register map

0x150000 - 0x15FFFF

MAC 10G

See Xilinx PG210 for register map

0x160000 - 0x16FFFF

MAC 10G

See Xilinx PG210 for register map

0x170000 - 0x17FFFF

MAC 10G

See Xilinx PG210 for register map

0x200000 - 0x20FFFF

Eth/IPv4 network stack for interface #0

0x210000 - 0x21FFFF

Eth/IPv4 network stack for interface #1

0x220000 - 0x22FFFF

Eth/IPv4 network stack for interface #2

0x230000 - 0x23FFFF

Eth/IPv4 network stack for interface #3

0x240000 - 0x24FFFF

Eth/IPv4 network stack for interface #4

0x250000 - 0x25FFFF

Eth/IPv4 network stack for interface #5

0x260000 - 0x26FFFF

Eth/IPv4 network stack for interface #6

0x270000 - 0x27FFFF

Eth/IPv4 network stack for interface #7

0x400000 - 0x47FFFF

64

Address table: decodes handles used by load_calibration and host_writer to DMA addresses

AXI Mailbox

AXI mailbox is used to send work request from host to action, and receive work completions. Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114).

Work request has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Work completion has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Special values:

65534 - start of data collection

65535 - end of data collection

15

31

Data collection ID

HBM memory

Interface number

Core

Meaning

0-1

jf_conversion

Gain factor G0

2-3

jf_conversion

Gain factor G1

4-5

jf_conversion

Gain factor G2

6-7

jf_conversion

Pedestal G0

8-9

jf_conversion

Pedestal G1

10-11

jf_conversion

Pedestal G2

12-13

integration

Integration map

14-15

integration

Integration weights

16-17

spot_finder_mask

Spot finder resolution

18-19

roi_calc

ROI calculation

20-21

frame_generator

Frame generator

22-27

load_from_hbm

Frame summation

\ No newline at end of file diff --git a/HARDWARE.html b/HARDWARE.html new file mode 100644 index 00000000..299a561a --- /dev/null +++ b/HARDWARE.html @@ -0,0 +1 @@ + Hardware requirements — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Hardware requirements

Operating Jungfraujoch requires the following:

  1. High performance server

  2. FPGA board(s) installed in the server

  3. (optionally) GPU boards

  4. (optionally) 100G switch to connect FPGA and the detector

Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include AMD FPGA boards. Therefore, the two has to be purchases separately. This might have impact on the warranty for the hardware and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn’t take any responsibility for warranty limitations for operating FPGA boards in the server. Having said this - we didn’t encounter any hardware issues so far.

High performance server

PSI is using HPE DL380 Gen11 servers are the moment to operate Jungfraujoch systems. However, this is because of general preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system from any other vendor with similar specification should work as well.

At PSI, we use the following configuration of HPE DL380 Gen11 to operate 9M pixel detectors at 2 kHz is as follows:

  • 2 x Intel Xeon 8558P

  • 512 GB RAM

  • 2 x Nvidia L4 GPU (for indexing)

  • 1 x Nvidia Connect-X 6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs)

  • Copper 1G/10G network

PCI slots

When ordering the system it is important to ensure enough PCIe cards can be accommodated in the system. In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2x GPU, 1x network

Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK.

FPGA

Jungfraujoch is built for AMD/Xilinx U55C (A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported.

Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires using TX delay functionality in the detector.

GPUs

Operating fast-feedback indexer code requires operation of a graphic processing unit from Nvidia. For practical reasons, i.e. power consumption and cost, we choose inference grade card Nvidia L4. In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work.

Network switch

Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs 8x10g variant of the Jungfraujoch FPGA image, which allows to directly connect 4 JUNGFRAU modules to one U55C card.

Such configuration is however impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujochs server. In this case one needs a network switch.

We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. For switches with only 100G ports it is important to ensure, that these can be split into 4x10G ports to connect the detector.

\ No newline at end of file diff --git a/HDF5.html b/HDF5.html new file mode 100644 index 00000000..5902ffca --- /dev/null +++ b/HDF5.html @@ -0,0 +1,18 @@ + HDF5 / NeXus data format — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

HDF5 / NeXus data format

Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be NXmx-compliant. On top of the NXmx application definition, Jungfraujoch records a substantial amount of derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and reusable.

This page documents the file layout and the data fields. The operational behaviour of the writer (running, republishing, file finalisation) is described in jfjoch_writer. The wire format that feeds the writer is described in CBOR messages; fields below frequently correspond one-to-one to CBOR message fields, and that document is a useful companion for their meaning.

1. Motivation: derived metadata and FAIR data

The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both rotation macromolecular crystallography (single- and multi-crystal, including fine-sliced and helical scans) and serial crystallography (stills, grid scans); the same concerns apply to both:

  • Findability. Raw diffraction images carry almost no descriptive metadata about content. Quantities such as background level, number of diffraction spots, or indexing outcome let a user judge the quality and relevance of a dataset before inspecting the raw images.

  • Accessibility at scale. A single experiment can span tens to hundreds of terabytes. Standard retrieval (e.g. HTTP) makes a dataset available but not inspectable — users would otherwise have to download a large fraction of the data just to decide whether it is useful. Compact derived representations make discovery, assessment and reuse feasible.

Because Jungfraujoch couples acquisition with real-time analysis used to steer experiments, transparency and reproducibility of that analysis matter. As a minimum the writer therefore preserves spot-finding and indexing results together with the filters that were applied, and it can retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse.

Two complementary layouts: per-image spots vs. a reflection table

Jungfraujoch stores analysis products in two shapes, matching how each is accessed.

Per-image spot finding / indexing. Spot finding and indexing are inherently image-centric — the natural query is “give me the spots for image n” — and this holds for serial stills and for rotation frames alike. For these products Jungfraujoch adopts a layout similar to the Coherent X-ray Imaging (CXI) data bank (Maia, 2012) and the convention understood by CrystFEL: spot properties (position, intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, with each image allocated room for up to a predefined maximum number of spots. These dense arrays are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved without traversing variable-length structures. The cost is some storage overhead for unused slots (padded with sentinels), which is acceptable for the access pattern.

Integrated reflections. Integrated intensities are naturally a dataset-wide table, which is exactly the model of the NeXus NXreflections base class. This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration results (see §4.2 below). We deliberately do not force spot finding/indexing into a single experiment-wide table: across the hundreds of thousands of patterns typical of serial — or fine-sliced rotation — experiments, that would require aggregating the whole experiment before the spots of one image can be read. We encourage the community to develop standardised NeXus application definitions for image-centric crystallography products that combine NeXus interoperability with the access patterns and scale of modern high-throughput experiments.

2. File layout

A run is written as one master file plus, depending on the format, one or more data files:

<prefix>_master.h5             # NXmx master file (metadata + links / virtual datasets)
+<prefix>_data_000001.h5        # data file: images + per-image analysis
+<prefix>_data_000002.h5
+...
+

The master file is produced by writer/HDF5NXmx.cpp; data files by writer/HDF5DataFile.cpp and its plugins (writer/HDF5DataFilePlugin*.cpp). Files are written to a temporary *.<random>.tmp name and renamed on successful close.

Three master-file variants exist (set via file_format):

Format

Value

Master ↔ data linking

NXmxLegacy (default)

1

One external link in /entry/data per data file (data_000001, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0.

NXmxVDS

2

A single virtual dataset /entry/data/data spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+.

NXmxIntegrated

3

No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format.

In legacy/VDS mode, image-indexed analysis arrays live in the data files and are exposed in the master file through external links or virtual datasets; in integrated mode they are written directly into the single file. Throughout this document a “✓ in master” column marks entries that are visible (directly or via link/VDS) from the master file.

Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or bitshuffle + Zstd; signed integer image datasets use INTx_MIN as the HDF5 fill value (the “masked / no-data” sentinel), unsigned use UINTx_MAX.

Reprocessing output: <prefix>_process.h5

The offline reprocessing tool rugnux (rugnux/rugnux_cli.cpp) re-runs the full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing dataset and writes its results to a master file named <prefix>_process.h5. This file uses the integrated format, but instead of copying the images its /entry/data/data is a virtual dataset that links back to the original image files (hdf5_source_data → NXmx::LinkToData_ProcessingVDS). The result is a compact, self-describing companion file that holds all the derived analysis (everything in §4) plus a virtual view of the raw images — without duplicating terabytes of data.

This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it processed, while the /entry/data/data VDS still resolves to the original images when they are available. rugnux can also process an equally-spaced subset of images (start/end/stride), producing a down-sampled reference set.

3. NXmx-standard content

The entries below are part of, or valid base classes for, the NXmx application definition. “NXmx” = listed in the application definition; “base” = a valid field of the relevant NeXus base class (NXdetector, NXsample, NXsource) but not in the NXmx required/recommended subset.

/entry (NXentry)

Field

Std

Notes

definition

NXmx

value "NXmx"

start_time

NXmx

arming time

end_time, end_time_estimated

NXmx

approximate end time

File-level HDF5 attributes file_name, file_time, HDF5_Version are also set.

/entry/source (NXsource), /entry/instrument (NXinstrument)

Field

Std

Units

source/name, source/type

NXmx / base

source/current

base

A

instrument/name

NXmx

/entry/instrument/beam (NXbeam)

Field

Std

Units

incident_wavelength

NXmx

angstrom

incident_wavelength_spread

NXmx

angstrom (only if polychromatic)

total_flux

NXmx

Hz

/entry/instrument/attenuator (NXattenuator)

Field

Std

attenuator_transmission

NXmx

/entry/instrument/detector (NXdetector)

Field

Std

Units

depends_on

NXmx

→ transformations/rot3

beam_center_x, beam_center_y

NXmx

pixel (0.0 = centre of the first pixel, see DETECTOR_GEOMETRY)

distance

NXmx

m

count_time, frame_time

NXmx

s

sensor_thickness

NXmx

m

sensor_material

NXmx

description

NXmx

threshold_energy

NXmx

eV (EIGER; written only for a single channel)

x_pixel_size, y_pixel_size

base

m

serial_number

base

bit_depth_readout

NXmx

saturation_value

NXmx

flatfield_applied

NXmx

pixel_mask, pixel_mask_applied

NXmx

pixel_mask is [y, x], hard-linked from detectorSpecific/pixel_mask

countrate_correction_applied

NXmx

number_of_cycles

base

frame-summation factor

/entry/instrument/detector/transformations (NXtransformations)

The NXtransformations mechanism (the depends_on chain, transformation_type, vector, offset attributes) is standard. The axis names follow the PyFAI PONI convention chosen by Jungfraujoch (see DETECTOR_GEOMETRY):

Axis

Type

Units

Depends on

translation

translation

m

.

rot1

rotation

rad

translation

rot2

rotation

rad

rot1

rot3

rotation

rad

rot2

The beam centre is encoded in translation (its offset from the sample), not only in the informational beam_center_x/beam_center_y fields. In a _process.h5 written by rugnux these axes carry the refined detector geometry — the refined beam centre folds into translation and the refined tilt into rot1/rot2/rot3; the broker writes the user-provided geometry unchanged.

/entry/instrument/detector/module (NXdetector_module)

data_origin, data_size, fast_pixel_direction, slow_pixel_direction, module_offset — all NXmx (fast/slow_pixel_direction and module_offset carry transformation attributes).

/entry/sample (NXsample)

Field

Std

Units / notes

name

NXmx

depends_on

NXmx

points at the last goniometer / grid-scan axis, or . for stills

temperature

NXmx

K

transformations/ (NXtransformations)

NXmx

rotation axis (e.g. omega) or grid-scan translation; hard-linked as /entry/sample/goniometer

unit_cell

base

[a, b, c, α, β, γ]

ub_matrix

base

[1, 3, 3], Angstrom⁻¹

For a rotation scan the goniometer axis is written as a per-image angle array <axis> plus <axis>_end, scalar <axis>_range_average, <axis>_range_total, and for helical scans <axis>_helical_x/_y/_z. These extra goniometer datasets beyond the bare axis array are Jungfraujoch conveniences.

/entry/data (NXdata)

data (3-D image stack, [n_images, y, x]) with image_nr_low / image_nr_high attributes. In legacy mode this group instead contains one external link data_000001, … per data file.

4. Extensions beyond NXmx

Everything in this section is outside the NXmx standard. Each group is declared with NX_class = NXcollection (the NeXus-sanctioned container for non-standardised content) unless noted. The per-image arrays are indexed by image number, padded to the run length and filled with a sentinel (NaN for floats, -1/0 for integer indices) where a quantity is absent.

4.1 /entry/MX — spot finding and indexing (CXI-style)

The flagship extension. Spot (“peak”) properties are stored as fixed-size [n_images, max_spots] arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as [n_images] vectors. In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master.

Per-spot arrays [n_images, max_spots]:

Dataset

Units

Meaning

Indexing only

peakXPosRaw, peakYPosRaw

pixel

spot position (raw detector frame)

peakTotalIntensity

photons

spot intensity

peakIceRingRes

spot lies in an ice-ring resolution band

peakH, peakK, peakL

Miller indices of the (indexed) spot

✓

peakDistEwaldSphere

Å⁻¹

distance of the spot from the Ewald sphere

✓

peakIndexed

spot fits the indexing solution

✓

peakLattice

lattice the spot belongs to (-1 = unindexed)

✓

Per-image vectors [n_images]:

Dataset

Units

Meaning

nPeaks

number of spots stored for the image (CXI)

strongPixels

strong-pixel count (first spot-finding stage)

peakCountUnfiltered

spots found before filtering

peakCountLowRes

low-resolution spots

peakCountIceRingRes

spots inside ice-ring bands

peakCountIceRingControl

spots in the ice-free flanks beside those bands, rescaled to their q width - the control for the count above (their ratio, pooled over the run, is the spot-based ice indicator)

peakCountIndexed

spots fitting the indexing solution

imageIndexed

image was indexed (0/1)

indexingLatticeCount

number of lattices found for the image

niggliClass

Niggli class of the indexed Bravais lattice (see International Tables for Crystallography A (2016), Vol. A, Table 3.1.3.1)

bravaisLattice

Bravais lattice short code, e.g. aP, mC, oF, tI, hP, hR, cF

profileRadius

Å⁻¹

crystal profile radius

mosaicity

deg

mosaicity estimate

bFactor

Ų

per-image B-factor estimate

resolutionEstimate

Å

diffraction resolution estimate

integratedReflections

number of integrated reflections

bkgEstimate

photons

mean background in the 3–5 Å resolution band

iceRingScore

ratio

strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

beam_corr_x, beam_corr_y

pixel

beam-center correction applied during processing

imageScaleFactor

on-the-fly per-image scale factor g

imageScaleCC

on-the-fly scaling correlation coefficient

imageScaleMosaicity

deg

scaling-model mosaicity

sweepQuality

why this image’s stretch of the sweep was flagged — see below

Per-image lattices: latticeIndexed [n_images, 9] (Å) — the real-space lattice (flattened 3×3); latticeIndexedExtra [n_images, max_extra_lattices, 9] (Å) — additional orientation variants.

Run-level summaries (written into the master /entry/MX at finalisation):

Dataset

Units

Meaning

indexing_algorithm

FFBIDX / FFT (CUDA) / FFT (FFTW)

geom_refinement_algorithm

e.g. beam_center

rotationLatticeIndexed

Å

whole-run rotation-indexing lattice ([9])

rotationLatticeIndexedExtra

Å

additional whole-run lattices ([m, 9])

rotationLatticeNiggliClass

Niggli class of the run lattice

imageIndexedMean

mean indexing rate over the run

bkgEstimateMean

photons

mean background over the run

iceRingScoreMean

ratio

mean iceRingScore over the run — the single “how icy was this dataset” number (1 = no ice)

indexedLatticeCount

per-image lattice count summary (master). Note: data files use indexingLatticeCount; readers accept either.

Sweep quality. sweepQuality [n_images] (uint8) says why the stretch of the sweep this image belongs to was flagged as delivering much less than the rest of the run: 0 means it was not, and any other value is a 1-based index into sweepQualityReasons, a string vector written beside it that carries the whole vocabulary, so the codes can be read without this source. The vocabulary is closed and stable — a code is never renamed and never reused — and currently reads no_diffraction, crystal_out_of_beam, weak_diffraction, loss_of_centring, radiation_damage; the rugnux documentation defines what each one means. Both datasets are absent unless the sweep-quality diagnostic ran, which needs scaling and merging; their absence therefore means “not looked for”, not “every image clean”. Written by the offline rugnux path only — the broker does not merge — and not carried on the CBOR stream, in the same way as the other offline-only fields (space_group_number, the refined geometry). Nothing is excluded from processing on the strength of it. The condensed, dataset-wide form of the same finding is in <prefix>_report.txt.

CrystFEL can read the spots directly with:

peak_list = /entry/MX
+peak_list_type = cxi
+

4.2 /entry/reflections — integrated reflections (NXreflections)

Integrated reflections are stored per image as /entry/reflections/image_NNNNNN groups, each declared NX_class = NXreflections. The columns map mostly onto the standard NXreflections base class:

Dataset

Units

NXreflections

Meaning

h, k, l

standard

Miller indices

d

Å

standard

resolution

int_sum

photons

standard

integrated intensity (summation)

int_err

photons

non-standard name

σ of the intensity (standard equivalent: int_sum_errors)

background_mean

photons

standard

mean background under the peak

background_variance

photons²

non-standard

non-signal part of σ², carried to the merge. Absent in files written before it existed; the reader then recovers it from σ² − I

predicted_x, predicted_y

pixel

name standard, units differ

predicted position. NXreflections predicted_x/_y are physical lengths; the pixel datasets are predicted_px_x/_y

observed_x, observed_y

pixel

name standard, units differ

observed centroid (pixels; standard pixel form is observed_px_x/_y)

observed_frame

standard

image number of the reflection

lp

standard

Lorentz–polarization factor (stored as 1/rlp)

partiality

standard

recorded fraction of the reflection

delta_phi

deg

extension

XDS Δφ: offset from the centre of the current frame

zeta

extension

Lorentz ζ factor (reciprocal-space geometry term)

image_scale_corr

extension

per-image scale correction; I_true = image_scale_corr · int_sum

In the master file these per-image groups are exposed through /entry/reflections external links (VDS/integrated formats).

4.3 /entry/azint — azimuthal integration

Dataset

Shape

Units

Meaning

bin_to_q

[φ_bins, q_bins]

Å⁻¹

q value of each bin

bin_to_two_theta

[φ_bins, q_bins]

deg

2θ of each bin

bin_to_phi

[φ_bins, q_bins]

deg

azimuthal angle of each bin

image

[n_images, φ_bins, q_bins]

per-image integrated profile (NaN for empty bins)

image_std

[n_images, φ_bins, q_bins]

per-bin standard deviation

image_count

[n_images, φ_bins, q_bins]

pixels contributing per bin

map

[y, x]

pixel→bin mapping (master file only)

4.4 /entry/roi — regions of interest (per-image results)

/entry/roi/<roi_name> has one sub-group per configured ROI, holding the per-image result vectors [n_images]. These are written into the data files; in VDS mode they are exposed from the master file through virtual datasets, and in integrated mode they are in the single file. (In legacy mode they remain only in the data files.)

Dataset

Meaning

max

maximum pixel value in the ROI

sum

sum of pixel values

sum_sq

sum of squared pixel values

npixel

number of valid pixels

x, y

intensity-weighted centroid

4.4.1 /entry/roi_defs — ROI definitions (master file)

The dataset-wide ROI definitions (geometry, fixed for the whole acquisition) live in the master file under a separate /entry/roi_defs group — kept apart from /entry/roi above so that older readers, which iterate /entry/roi, are unaffected by these entries. One sub-group /entry/roi_defs/<roi_name> per ROI:

Dataset

Meaning

bit_index

which bit of roi_map (below) marks this ROI

type

box, circle or azim

min_x_pxl, max_x_pxl, min_y_pxl, max_y_pxl

box bounds (type box)

center_x_pxl, center_y_pxl, radius_pxl

circle (type circle)

q_min_recipA, q_max_recipA

Q range (type azim)

phi_min_deg, phi_max_deg

azimuthal-angle sector (type azim, omitted for a full ring)

/entry/roi_defs/roi_map [y, x] is a uint16 per-pixel bitmask: bit bit_index is set for every pixel belonging to that ROI, so an ROI’s footprint can be recovered exactly.

4.5 /entry/image — per-image pixel statistics

[n_images] vectors: max_value, min_value (viable min/max, excluding error/saturated pixels), error_pixels, saturated_pixels, pixel_sum. Surfaced in the master file under /entry/image.

4.6 /entry/profiling — per-image timing

[n_images] vectors in seconds: spotFindingTime, indexingTime, integrationTime, refinementTime, processingTime, braggPredictionTime, preprocessingTime, compressionTime, azIntTime, indexAnalysisTime, imageScaleTime.

4.7 /entry/detector — acquisition diagnostics (data file)

A convenience NXcollection in the data file (note: distinct from the standard /entry/instrument/detector). In integrated format these datasets are written under /entry/instrument/detector/detectorSpecific instead.

Dataset

Meaning

timestamp, exptime

per-image timestamp and exposure time

number

image number (original number if image rejection was used)

det_info

JUNGFRAU debug field

storage_cell_image

storage-cell number

rcv_delay, rcv_free_send_buffers

receiver internal diagnostics

packets_expected, packets_received

UDP packets per image

data_collection_efficiency_image

received / expected packet ratio

4.8 /entry/xfel — pulsed-source metadata

[n_images] vectors pulseID and eventCode, written for pulsed sources (e.g. SwissFEL).

4.9 Other collections

Path

Class

Content

/entry/instrument/detector/detectorSpecific

NXcollection

Dectris-style detector metadata + Jungfraujoch fields: x_pixels_in_detector, y_pixels_in_detector, nimages, ntrigger, nimages_collected, nimages_written, data_collection_efficiency, max_receiver_delay, storage_cell_number, storage_cell_delay [ns], software_git_commit, software_git_date, jfjoch_release, jfjoch_writer_release, summation_mode, detect_ice_rings, gain_file_names, data_reduction_factor_serialmx, adu_histogram/, data_collection_efficiency_image

/entry/instrument/detector/calibration

NXcollection

per-channel pedestal / calibration images (bitshuffle-compressed)

/entry/instrument/fluorescence

NXcollection

XRF spectrum: energy [eV], data

/entry/user

NXcollection

scalar values supplied under header_appendix.hdf5

4.10 Non-standard fields inside the NXmx detector group

A few extension scalars are written inside the otherwise-standard /entry/instrument/detector group for compatibility with existing tooling:

Field

Units

Meaning

detector_distance

m

duplicate of distance (Dectris/Neggia compatibility)

detector_number

detector identifier (Dectris convention)

error_value

masked/error pixel sentinel (NXmx standard would be underload_value)

bit_depth_image

stored image bit depth (NXmx standard is bit_depth_readout)

acquisition_type

always triggered (Dectris convention)

jungfrau_conversion_applied

JUNGFRAU photon/keV conversion applied

jungfrau_conversion_factor

eV

conversion factor

geometry_transformation_applied

module→full-detector geometry applied

4.11 User-supplied metadata: header_appendix and image_appendix

Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two free-form JSON fields in the /start request (broker/jfjoch_api.yaml) provide this without any schema change; both accept any valid JSON:

Field

Carried in

Persisted to HDF5?

header_appendix

the start message, under user_data.user (see CBOR)

no — except the hdf5 sub-object (below)

image_appendix

every image message, as user_data

no

Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, republished analysis, viewers), so they are the recommended channel for facility- or beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has no dedicated API field.

Persisting selected values to HDF5. header_appendix is normally not written to the master file. As an exception, if it contains a key hdf5 whose value is a JSON object of scalars (strings and numbers — no arrays or nested objects), the writer stores each entry under /entry/user/<key>.

For example, a /start request containing:

{
+  "header_appendix": {
+    "proposal": "p20001",
+    "operator": "jdoe",
+    "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 }
+  },
+  "image_appendix": { "trigger_source": "external" }
+}
+

forwards the whole header_appendix as user_data.user on the start message and {"trigger_source": "external"} as user_data on every image message, and writes three scalars into the master file:

/entry/user/beamline          = "X06SA"
+/entry/user/ring_mode         = "top-up"
+/entry/user/attenuator_foils  = 2
+

5. Notes

  • Units are written as the HDF5 units attribute on the dataset (e.g. m, eV, deg, Angstrom, Angstrom^-1, Angstrom^2, pixel, s).

  • Sentinels. Missing per-image values are NaN (floats) or -1/0 (integer indices); image pixels use INTx_MIN / UINTx_MAX.

  • Master vs data file. In legacy/VDS formats the analysis arrays physically live in the data files; the master file links to them (external links in legacy, virtual datasets in VDS). In the integrated format there are no data files and everything is in one place.

  • CXI / CrystFEL. /entry/MX follows the CXI peak-list convention; see CXI file format.

\ No newline at end of file diff --git a/IMAGE_STREAM.html b/IMAGE_STREAM.html new file mode 100644 index 00000000..a6e57cc7 --- /dev/null +++ b/IMAGE_STREAM.html @@ -0,0 +1,30 @@ + Data streams — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Data streams

Jungfraujoch process (jfjoch_broker) operates three outputs. All three can be operated/enabled independently. These are:

  • Image - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket)

  • Preview - images with metadata at a reduced frame rate (PUB socket)

  • Metadata - only metadata for all the images, bundled into packages (PUB socket)

Image stream

Images (with metadata) are serialized as CBOR image message. The stream will also include CBOR start message, calibration messages and end message with run metadata.

If file_prefix is not provided for a data collection, images won’t be sent to image stream (or its HDF5/CBOR replacements).

Splitting image stream

Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. All sockets will forward start and end messages. Only first socket will forward calibration messages and will be marked to write master file.

ZeroMQ image stream

This is using PUSH ZeroMQ socket(s). It should be strictly avoided to have multiple receivers connected to one PUSH ZeroMQ socket. ZeroMQ will send the images in a round-robin basis to the receivers. In this case start and end messages will end up only with one receiver. Instead, Jungfraujoch feature of multiple sockets should be used. For ZeroMQ image stream, each writer connects to a different port.

Behavior is as following:

  • Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer.

  • Calibration message is sent to the first socket only, with timeout of 1s.

  • Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped).

  • End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded.

The format is generally interchangeable with DECTRIS Stream2 format.

ZeroMQ configuration

ZeroMQ image stream is configured in the broker JSON configuration file under the zeromq_settings section:

{
+  "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"],
+  "send_watermark": 100,
+  "send_buffer_size": 67108864,
+  "writer_notification_socket": "tcp://192.168.0.1:*"
+}
+

  • image_socket: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (tcp://, ipc://). 0.0.0.0 binds on all network interfaces.

  • send_watermark (optional): ZeroMQ send high-water mark (number of outstanding messages per socket).

  • send_buffer_size (optional): OS-level send buffer size for the ZeroMQ socket.

  • writer_notification_socket (optional): see Writer notification socket below.

TCP/IP image stream

This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication between the data collection and the writer, and is therefore more robust than ZeroMQ.

For TCP/IP image stream, Jungfraujoch listens on a single TCP port and all writers connect to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic KEEPALIVE frames when no data collection is active to detect dead connections; writers are expected to respond with a KEEPALIVE pong.

Using * as port number (e.g. tcp://127.0.0.1:*) is supported — the OS assigns a free port and the actual bound address can be queried via GetAddress().

Payloads for START, DATA, CALIBRATION and END frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages.
ACK, CANCEL, and KEEPALIVE are control frames (no CBOR payload).

The data collection lifecycle on each connection follows: START → CALIBRATION (socket 0 only) → DATA (repeated) → END

If a START ACK fails on any connection, Jungfraujoch sends CANCEL to all already-started connections and rolls back.

For each frame:

  1. Read one TcpFrameHeader (fixed size, 64-byte aligned).

  2. Validate magic (0x4A464A54 / "JFJT") and version (2).

  3. Read payload_size bytes (if non-zero).

When image stream is split into multiple connections:

  • START and END are sent on all connections,

  • CALIBRATION is sent only on connection 0,

  • DATA frames are distributed by file grouping: connection index = (image_number / images_per_file) % num_connections.

TCP/IP configuration

TCP/IP image stream is configured in the broker JSON configuration file under the tcp_settings section:

{
+  "image_socket": "tcp://192.168.0.1:9100",
+  "nwriters": 2,
+  "send_buffer_size": 67108864
+}
+

  • addr: listen address in tcp://<IP>:<port> format. 0.0.0.0 binds on all interfaces. * as port selects a random free port.

  • nwriters (optional): maximum number of simultaneous writer connections accepted.

  • send_buffer_size (optional): OS-level SO_SNDBUF size for accepted connections.

ACK handling

ACK handling is mandatory for correct operation:

  • START must be acknowledged (ACK with ack_for=START) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered.

  • END must be acknowledged (ack_for=END) on each connection within 10 seconds for successful completion.

  • CANCEL should be acknowledged during rollback paths (500ms timeout).

  • DATA should be acknowledged for every frame. A DATA ACK with FATAL flag set reports a downstream error (e.g. disk full) which is propagated to jfjoch_broker via Finalize(). A failed DATA ACK does not break the TCP connection on its own — data continues to flow.

  • CALIBRATION is not acknowledged at this time.

  • KEEPALIVE frames are not acknowledged via ACK; the writer responds with a KEEPALIVE pong frame instead.

Keepalive

When no data collection is active, Jungfraujoch sends KEEPALIVE frames approximately every 5 seconds on each persistent connection. Writers should respond with a KEEPALIVE frame (pong). OS-level TCP keepalive is also enabled (TCP_KEEPIDLE=30s, TCP_KEEPINTVL=10s, TCP_KEEPCNT=3) as a secondary safety net. Dead connections are automatically removed from the pool.

Zero-copy transmission

On Linux, large payload transmission (DATA and CALIBRATION frames) can use kernel TCP zero-copy (SO_ZEROCOPY/MSG_ZEROCOPY) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal send() behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread.

Frame types

Value

Name

Purpose

1

START

Start-of-run metadata

2

DATA

One image payload

3

CALIBRATION

Calibration payload

4

END

End-of-run metadata

5

ACK

Acknowledgement / error reporting

6

CANCEL

Cancel run initialization/stream

7

KEEPALIVE

Connection liveness probe/pong

TCP frame header (TcpFrameHeader)

Field

Type

Description

magic

uint32_t

Protocol magic (0x4A464A54, "JFJT")

version

uint16_t

Protocol version (2)

type

uint16_t

Frame type (see table above)

image_number

uint64_t

Image index for DATA frames

payload_size

uint64_t

Number of payload bytes after header

socket_number

uint32_t

Connection index in split-stream mode

flags

uint32_t

ACK flags (OK, FATAL, HAS_ERROR_TEXT)

run_number

uint64_t

Run identifier

ack_processed_images

uint32_t

In ACK: number of images processed by receiver

ack_code

uint16_t

In ACK: error/status code

ack_for

uint16_t

In ACK: frame type being acknowledged

ack_fifo_occupancy

uint16_t

In ACK: occupancy of input FIFO in the jfjoch_writer

ack_fifo_max_occupancy

uint64_t

In ACK: max occupancy of input FIFO

The header is 64-byte aligned (alignas(64)).

ACK semantics

  • ACK frames use ack_for to indicate which frame type is acknowledged.

  • flags:

    • OK (bit 0): operation accepted/successful,

    • FATAL (bit 1): receiver reports unrecoverable error (primarily for DATA),

    • HAS_ERROR_TEXT (bit 2): ACK payload contains UTF-8 error text.

  • ack_code can be used to categorize errors:

Code

Name

Meaning

0

None

No error

1

StartFailed

START processing failed

2

DataWriteFailed

Image write failed

3

EndFailed

END processing failed

4

DiskQuotaExceeded

Disk quota exceeded

5

NoSpaceLeft

No space left on device

6

PermissionDenied

Permission denied

7

IoError

General I/O error

8

ProtocolError

Protocol-level error

Image stream replacement

Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting “None” image pusher for all the measurements.

Writer notification socket

The writer notification socket is used only with ZeroMQ image stream. Since ZeroMQ is asynchronous, jfjoch_broker does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back.

For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame.

To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with writer_notification_socket entry:

{
+  "writer_notification_socket":"tcp://192.168.0.1:*"
+}
+

Such entry will create PULL socket on 192.168.0.1 network interface listening on one, random TCP port. When data processing is started, the image stream will send CBOR start message. This message will include information on writer_notification_zmq_addr, which needs to be used by downstream code. Since the start message must reference the address of jfjoch_broker host, notification socket should always listen on a particular network interface, and should not be configured with placeholder address 0.0.0.0. It is, however, OK to use placeholder :* for network port, as it will be substituted for the one chosen by ZeroMQ.

For every image stream socket, downstream code must send the following message to the PULL socket:

{
+  "run_number":135,
+  "run_name": "sample_1",
+  "socket_number": 1,
+  "processed_images":250,
+  "ok": true
+}
+

Here run_number, run_name and socket_number must match information from the start message. ok is boolean confirming if the writing process was OK. processed_images is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. If not, it is possible to include error message:

{
+  "run_number":135,
+  "run_name": "sample_1",
+  "socket_number": 1,
+  "processed_images": 0,
+  "ok": false,
+  "error": "Permission error"
+}
+

This way errors from the downstream code are propagated to jfjoch_broker.

If writer notification socket is configured, but downstream code doesn’t send proper notification, jfjoch_broker will time out after 60 seconds producing an error message.

Preview stream

Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. Images are serialized as CBOR image message. The stream will also include CBOR start message and end message with run metadata. Only start and image messages are sent.

This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope with the messages, it will always receive the last generated message (no backlog). For this reason it is also recommended to use the same option on receiver side.

Given PUB socket properties, it is possible to connect multiple viewers to a single socket — all the viewers should receive all the images sent.

Metadata stream

Jungfraujoch can also send pure metadata for the purpose of archiving such information. Metadata are serialized as CBOR metadata message. This is very similar as image message, but excludes the actual image array and spot positions. As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. Order of images within bundle, as well a size of the bundle, are not guaranteed. The stream will also include CBOR start message and end message with run metadata.

This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected.

\ No newline at end of file diff --git a/JFJOCH_BROKER.html b/JFJOCH_BROKER.html new file mode 100644 index 00000000..364d3dc9 --- /dev/null +++ b/JFJOCH_BROKER.html @@ -0,0 +1,126 @@ + jfjoch_broker — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

jfjoch_broker

jfjoch_broker is the main service for the Jungfraujoch application. It is responsible for:

  • Providing user interface via HTTP and OpenAPI

  • Configuring FPGA firmware

  • Building images from FPGA output and forwarding the results over ZeroMQ

External interfaces

Broker operates four external interfaces.

Image stream ZeroMQ PULL socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream processing. See details here.

Preview stream ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details here.

Metadata stream ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details here.

Configuration, status and results interface HTTP/REST interface described in the OpenAPI format. Description of the API is presented in the OpenAPI specification.

Broker configuration

jfjoch_broker requires JSON configuration files. The file is described by OpenAPI structure jfjoch_settings defined in jfjoch_api.yaml file. It is recommended to go through example files in the etc/.

Example with all fields:

{
+  "pcie": [
+    {
+      "blk": "/dev/jfjoch0",
+      "ipv4": "10.1.1.7"
+    },
+    {
+      "blk": "/dev/jfjoch1",
+      "ipv4": "10.1.1.8"
+    }
+  ],
+  "zeromq": {
+    "send_watermark": 100,
+    "send_buffer_size": 1024,
+    "image_socket": [
+      "tcp://1.2.3.4:5000",
+      "tcp://1.2.3.4:5001"
+    ],
+    "writer_notification_socket": "tcp://1.3.4.6:7000"
+  },
+  "instrument": {
+    "source_name": "Swiss Light Source",
+    "source_type": "Synchrotron X-ray Source",
+    "instrument_name": "X06SA",
+    "pulsed_source": false,
+    "electron_source": false
+  },
+  "detector": [
+    {
+      "description": "EIGER 1M",
+      "serial_number": "E1M-01",
+      "type": "EIGER",
+      "high_voltage_V": 150,
+      "udp_interface_count": 1,
+      "module_sync": true,
+      "sensor_thickness_um": 320,
+      "calibration_file": [
+        "gainMaps.bin"
+      ],
+      "hostname": [
+        "e1m-01",
+        "e1m-02"
+      ],
+      "readout_time_us": 3,
+      "sensor_material": "Si",
+      "tx_delay": [
+        0,1
+      ],
+      "base_data_ipv4_address": "10.10.10.50",
+      "standard_geometry": {
+        "nmodules": 1,
+        "gap_x": 8,
+        "gap_y": 36,
+        "modules_in_row": 1
+      },
+      "custom_geometry": [
+        {
+          "x0": 0,
+          "y0": 0,
+          "fast_axis": "Xp",
+          "slow_axis": "Xp"
+        }
+      ],
+      "mirror_y": true
+    }
+  ],
+  "detector_settings": {
+    "frame_time_us": 450,
+    "count_time_us": 0,
+    "internal_frame_generator": false,
+    "internal_frame_generator_images": 1,
+    "detector_trigger_delay_ns": 0,
+    "timing": "auto",
+    "eiger_threshold_keV": 6.0,
+    "jungfrau_pedestal_g0_frames": 2000,
+    "jungfrau_pedestal_g1_frames": 300,
+    "jungfrau_pedestal_g2_frames": 300,
+    "jungfrau_pedestal_g0_rms_limit": 100,
+    "jungfrau_pedestal_min_image_count": 128,
+    "jungfrau_storage_cell_count": 1,
+    "jungfrau_storage_cell_delay_ns": 5000,
+    "jungfrau_fixed_gain_g1": false,
+    "jungfrau_use_gain_hg0": false
+  },
+  "azim_int": {
+    "polarization_factor": -1,
+    "solid_angle_corr": true,
+    "high_q_recipA": 0,
+    "low_q_recipA": 0,
+    "q_spacing": 0
+  },
+  "image_format": {
+    "summation": true,
+    "geometry_transform": true,
+    "jungfrau_conversion": true,
+    "jungfrau_conversion_factor_keV": 0.001,
+    "bit_depth_image": 16,
+    "signed_output": true,
+    "mask_module_edges": true,
+    "mask_chip_edges": true
+  },
+  "image_buffer_MiB": 2048,
+  "receiver_threads": 64,
+  "frontend_directory": "/usr/share/jfjoch/frontend",
+  "image_pusher": "ZeroMQ",
+  "zeromq_metadata": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4357"
+  },
+  "zeromq_preview": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4356"
+  }
+}
+

Setting up a local test for Jungfraujoch

For development, it is possible to set up a local installation of Jungfraujoch. This will work without FPGA installed in the computer and allows to test Jungfraujoch software layer, including ZeroMQ streaming and file writing.

The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore very low, as fixed-point calculations have large performance penalty on CPU. In the CPU simulation mode, one can simulate using only a single FPGA device.

To run the test:

Compile Jungfraujoch with frontend

mkdir build
+cd build
+cmake ..
+make jfjoch
+make frontend
+

Alternatively, for RHEL8 system, you can use RPM generated by automated pipeline. Solely jfjoch one is enough. In this case - it is necessary to update etc/broker_local.json file with frontend path in /usr/share/jfjoch/frontend.

Start service

Start broker:

cd build/broker
+./jfjoch_broker ../../etc/broker_local.json 5232
+

Run tests

To run test a Python script is provided:

cd tests/test_data
+python jfjoch_broker_test.py
+

The script will initialize Jungfraujoch, import test image and start data collection.

Expected result

You can observe online data analysis by opening the following web page: http://localhost:5232. Also, a dataset with images should be written in the build/broker directory.

\ No newline at end of file diff --git a/JFJOCH_VIEWER.html b/JFJOCH_VIEWER.html new file mode 100644 index 00000000..455390b0 --- /dev/null +++ b/JFJOCH_VIEWER.html @@ -0,0 +1,17 @@ + jfjoch_viewer — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

jfjoch_viewer

jfjoch_viewer is the interactive desktop application of Jungfraujoch. It opens diffraction datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal integration, per-image statistics), and can follow a live data collection by syncing with a running jfjoch_broker over its HTTP interface.

It is a standalone Qt 6 application, distributed pre-built for Linux and Windows on the Gitea release page and in the Jungfraujoch RPM/APT repositories — see Release contents for what each package contains and what it requires, and Deployment for how to install it.

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

On screen; a processing job can write the same files as rugnux

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Functionality

  • Opens HDF5 files written by jfjoch_writer (*_master.h5) and the *_process.h5 files produced by rugnux. It also opens NXmx files written by DECTRIS detectors, though that path has had only limited testing.

  • Runs an embedded data-processing pipeline — the same analysis code as the rest of Jungfraujoch — performing spot finding, indexing and integration on the displayed image, with the result drawn over it. This interactive analysis is not written anywhere.

  • Runs full processing jobs on the open dataset with Analyze dataset, on the same rugnux engine and off the GUI thread. The settings panel’s MX / AzInt / Calib toggle decides what a run does — full analysis, azimuthal integration only, or a detector calibration — over a chosen image range, optionally writing _process.h5 and the merged .mtz/.cif. A finished run becomes a selectable view of the dataset, so several processing runs can be compared against each other, and its merging statistics (or, for a calibration, its fitted geometry) open in their own window; the Processing panel lists the runs and reopens those results. The equivalent rugnux command line can also be copied out to run the same job on a cluster instead.

  • Detector calibration against a powder standard, on the Calib page: pick the calibrant (LaB6, AgBh, CeO2, Si, ice, or the open dataset’s own unit cell) and fit either the image on screen (Guess / Refine detector calibration) or the whole dataset (Analyze dataset, which writes a pyFAI <output prefix>.poni). The whole-dataset fit measures the rings either from the azimuthally-binned profile summed over the run (Rings, the default) or from the pooled spot lists (Spots), and reports PONI x/y, the two tilts and the distance against the header values. Judge it by the radial rms, not the beam-centre sigma: the sigma shrinks with the number of ring points, so a fit that sits a couple of pixels off every ring can still report a small one. Rings needs the run to be integrated in azimuthal sectors — with the AzInt page’s Azimuthal bins below 4 the calibration run raises it to 32, as rugnux --mode calibration does, and says so.

  • Settings panel for the geometry, unit cell, spot finding, indexing, azimuthal integration, Bragg integration, scaling, powder calibration and a reference dataset — the same settings the CLI takes.

  • Auxiliary windows: image list, dataset metadata, spot list, reflection list, reciprocal-space viewer, 2D azimuthal-integration image, calibration-image viewer and a magnifier; plus the Inspector (per-image statistics, image features, resolution rings, ROI statistics), the Image strip thumbnail feed and dataset-info charts.

  • User-mask editing: build a user mask interactively, load one from TIFF (replacing or adding to the current one), save it as TIFF, clear it, or upload it to a connected server.

  • Layout presets (View ▸ Image layout / Processing layout / Reset layout) rearrange the docks for looking at images or at processing results.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. On systems with a GPU, use the CUDA build (a separate package variant everywhere: RPM/APT repository, .tgz and Windows installer) for the embedded indexing and integration; the non-CUDA build runs the same pipeline on the CPU at much lower throughput. The CUDA build also runs on a machine without a GPU — see Release contents ▸ CUDA and non-CUDA builds.

Opening data

  • File ▸ Open (Ctrl+O) — open a local HDF5 file.

  • File ▸ Open HTTP (Ctrl+H) — connect to a jfjoch_broker HTTP endpoint to follow a live collection. The dialog defaults to host localhost and port 8080; these defaults can be overridden with the environment variables JUNGFRAUJOCH_HTTP_HOST and JUNGFRAUJOCH_HTTP_PORT.

  • Command line — jfjoch_viewer <file.h5> opens a file (or an http://host:port URL) on start-up. --dbus <true|false> (-d) enables or disables the D-Bus interface (default: enabled); --help and --version behave as usual.

D-Bus interface

When enabled, the viewer registers the D-Bus interface ch.psi.jfjoch_viewer, so other processes can drive it:

  • LoadFile(filename, image_number=0, summation=1) — open a file (or an http://host:port URL) and display the given image.

  • LoadImage(image_number, summation=1) — navigate to an image in the already-open dataset.

summation sums that many consecutive images before display.

Building from source on Windows

jfjoch_viewer is the one Jungfraujoch component that is cross-platform: it builds on Windows 11 with MSVC and the full CUDA GPU path. (The rest of Jungfraujoch — broker, receiver, FPGA host — is Linux-only.) A pre-built installer is published with every release, so building from source is only needed to develop or to change the build options. On Windows the build is automatically restricted to the viewer and the libraries it needs (JFJOCH_VIEWER_ONLY is forced on), and the remaining dependencies are fetched and built automatically (the first configure needs network access).

Verified toolchain — the same one the released installer is built with:

  • Windows 11

  • Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC

  • CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path

  • Qt 6.11 for MSVC (msvc2022_64), including the Qt Charts module — e.g. C:\Qt\6.11.1\msvc2022_64

  • CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of the box — it comes with the C++ workload, so there is nothing extra to install. Any recent standalone CMake (from cmake.org, or the one bundled with Qt in C:\Qt\Tools\CMake_64) works too.

  • zlib and Eigen — the two libraries not auto-fetched on Windows. Build/install both into one prefix (here C:\deps) and point CMake at it:

    :: static zlib
+git clone --branch v1.3.1 https://github.com/madler/zlib
+cmake -G Ninja -S zlib -B zlib-build -DCMAKE_INSTALL_PREFIX=C:/deps
+cmake --build zlib-build --target install
+:: Eigen 3.4 (header-only) -- install just the headers with `cmake --install`; the BLAS/LAPACK/test
+:: targets are disabled since they are not needed (and fail to build under MSVC). Use the 3.4 series:
+:: the project requests find_package(Eigen3 3.4), which Eigen's same-major rule rejects for 5.x.
+git clone --branch 3.4.0 https://gitlab.com/libeigen/eigen.git
+cmake -G Ninja -S eigen -B eigen-build -DCMAKE_INSTALL_PREFIX=C:/deps ^
+  -DEIGEN_BUILD_BLAS=OFF -DEIGEN_BUILD_LAPACK=OFF -DEIGEN_BUILD_DOC=OFF -DBUILD_TESTING=OFF
+cmake --install eigen-build
+

  • Optional: NSIS to build the .exe installer.

Configure and build from an x64 Native Tools Command Prompt for VS 2026 (so cl, nvcc and ninja are on PATH):

cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^
+  -DCMAKE_PREFIX_PATH="C:/deps;C:/Qt/6.11.1/msvc2022_64"
+cmake --build build-win --target jfjoch_viewer
+

Notes:

  • CMAKE_PREFIX_PATH (the C:/deps prefix plus Qt) is the only required flag — CMake finds zlib and Eigen from the prefix, so no separate -DZLIB_ROOT is needed.

  • The CUDA toolchain is located automatically from the CUDA_PATH environment variable that the CUDA installer sets (or from nvcc on PATH). Pass -DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe" only if nvcc is installed in a nonstandard location and is not found.

  • For a machine without an NVIDIA GPU, add -DJFJOCH_USE_CUDA=OFF: the viewer then runs the same pipeline on the CPU (FFTW indexer) at lower throughput.

To produce a self-contained installer (bundles the Qt runtime via windeployqt, the analysis CLIs, and — on the CUDA build — the cuFFT runtime DLL, so the target host needs neither Qt nor a CUDA toolkit), with NSIS installed:

cd build-win
+cpack
+

The NSIS generator is selected automatically on Windows (no -G needed). What comes out, and how the CUDA and CPU variants are named and told apart, is described in Release contents ▸ Windows installer.

\ No newline at end of file diff --git a/JFJOCH_WRITER.html b/JFJOCH_WRITER.html new file mode 100644 index 00000000..2c909832 --- /dev/null +++ b/JFJOCH_WRITER.html @@ -0,0 +1,71 @@ + jfjoch_writer — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

jfjoch_writer

jfjoch_writer is NeXus compliant HDF5 file writer.

Acknowledgements

  • Zdenek Matej (MAX IV)

  • Felix Engelmann (MAX IV) for testing and multiple improvement suggestions.

Running directory

Writer needs to be running in base directory for writing files - file_prefix will be always relative in regard to writer running directory. Writer detects and protects for basic security issues, like file_prefix starting with a slash, or starting with ../, or containing /../.

Usage

Writer needs to be started as a background service, with the following command:

jfjoch_writer {options} <address to connect via ZeroMQ to DCU>
+
+Options:
+-T        | --tcp                      Use raw TCP/IP instead of ZeroMQ
+-j<int>   | --nproc=<int>              Number of forks (only with -T)
+-d<path>  | --root_dir=<path>          Root directory for file writing (-R is a deprecated alias)
+-r<int>   | --zmq_repub_port=<int>     ZeroMQ port for PUSH socket to republish images
+-f<int>   | --zmq_file_port=<int>      ZeroMQ port for PUB socket for notifications on finalized files
+-w<int>   | --rcv_watermark=<int>      Receiving ZeroMQ socket watermark (default = 100)
+-W<int>   | --repub_watermark=<int>    Republish ZeroMQ socket watermark (default = 1000)
+-v        | --verbose                  Verbose output
+-h                                     This message
+

for example:

jfjoch_writer -d /data tcp://dcu-address:5400 
+

Status and cancellation

When a data collection is finalized, each writer reports its outcome back to jfjoch_broker over the writer notification socket — a ZeroMQ address the broker passes in the START message (writer_notification_socket in the broker configuration) — as a JSON message with the socket number, run name and number, processed image count, throughput, and on failure an error string. That is how the broker learns that a writer could not write. On the TCP/IP image stream, failures additionally come back in-band as negative acknowledgements (see Data streams).

To stop a writer, send it SIGINT, SIGQUIT, SIGTERM or SIGHUP: it closes the HDF5 files it is writing and exits. This is only for the case where the broker was terminated or disconnected — it is not the normal way to end a data collection, which the broker finishes on its own.

Republish

Republish creates a PULL socket on the writer, where all the messages are republished for further use by data analysis pipeline. Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won’t be republished. In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won’t be retried.

Republish functionality is optional, if republish port number is omitted this functionality is not enabled.

Overwriting files

When jfjoch_writer creates a HDF5 file, it first adds suffix .<random>.tmp. Random value depends on current time-stamp and likely will be different from each file of the particular series. After file is all saved and closed, it is renamed to remove the suffix. By default, renaming won’t happen if this would overwrite existing file. However, this behavior can be changed by setting overwrite parameter to true in the file writer configuration.

When the overwrite conflict is reported

An existing output file is a fatal condition (unless overwrite is true). When it is detected depends on whether the transport between the broker and the writer has a back-channel to report the failure before acquisition starts:

  • Direct HDF5 pusher and TCP writer (back-channel available). The conflict is detected at start: the writer that owns the master file checks whether it already exists and refuses to start. The direct pusher raises the error in-process; the TCP writer returns a START-failure acknowledgement. Either way the broker learns immediately and aborts the data collection before the detector is armed — no images are taken and nothing is written. Only the master file is checked up front: in a multi-writer setup the per-image data files are staggered across writers, and checking them at start would make each writer inspect files it never writes (and race the writers that do). Data-file conflicts are instead caught by their owning writer at the final rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker.

  • ZeroMQ writer (no back-channel). The ZeroMQ image stream is fire-and-forget: the writer has no way to tell the broker to stop, and the broker would keep streaming images regardless. The writer therefore does not fail at start. It writes the whole series to the .<random>.tmp files as usual and only fails at the final rename, leaving the .tmp files on disk. This is deliberate: the acquired images are preserved (in .tmp form) rather than being dropped by a writer that aborted mid-stream. Rename the .tmp files by hand, or re-run with overwrite set, to recover them.

Finalized files information

Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure:

{
+  "filename": <string>: HDF5 data file name (relative to writer root directory),
+  "nimages": <int> number of images in the file (counting from 1!),
+  "file_number": <int> number of file within the acquisition,
+  "sample_name": <string> name of sample,
+  "run_name": <string> name of run,
+  "run_number": <int> number of run,
+  "experiment_group": <string> number of p-group / proposal (optional),
+  "user_data": <any json> user_data,
+  "beam_x_pxl": <float> beam center (X) in pixels,
+  "beam_y_pxl": <float> beam center (Y) in pixels,
+  "detector_distance_m": <float> detector distance (X) in m,
+  "detector_height_pxl": <int> detector size (X) in pixels,
+  "detector_width_pxl": <int> detector size (Y) in pixels,
+  "incident_energy_eV": <float> photon energy of the X-ray beam,
+  "pixel_size_m": <float> pixel size in meter (assuming pixel X == Y),
+  "saturation": <int> this count and higher mean saturation,
+  "space_group_number": <int> space group number (optional),
+  "underload": <int> pixels with this count should be excluded,
+  "unit_cell": <optinal> unit cell dimensions in Angstrom/degree {
+    "a": <float>, "b": <float>, "c": <float>,
+    "alpha": <float>, "beta": <float>, "gamma": <float>
+  },
+}
+

user_data is defined as header_appendix in the /start operation in the jfjoch_broker. Other metadata are also carried over from /start operation.

If the header_appendix is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. For example header_appendix of {"param1": "test1", "param2": ["test1", "test2"]}, than example message will look as follows:

{
+  "filename": "dataset_name_data_000001.h5",
+  "nimages": 1000,
+  "file_number": 0,
+  "sample_name": "my_sample",
+  "run_name": "my_run",
+  "run_number": 25,
+  "experiment_group": "p00001",
+  "beam_x_pxl": 1200,
+  "beam_y_pxl": 1500,
+  "detector_distance_m": 0.155,
+  "detector_height_pxl": 2164,
+  "detector_width_pxl": 2068,
+  "image_time_s": 0.001,
+  "nimages": 2,
+  "incident_energy_eV": 12400.0,
+  "pixel_size_m": 7.5e-05,
+  "saturation": 32766,
+  "space_group_number": 96,
+  "underload": -32768,
+  "unit_cell": {
+    "a": 78.0,
+    "alpha": 90.0,
+    "b": 78.0,
+    "beta": 90.0,
+    "c": 39.0,
+    "gamma": 90.0
+  },
+  "user_data": {
+    "param1": "test1", 
+    "param2": ["test1", "test2"]
+  }
+}
+

Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled.

HDF5 file structure

Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics and timing) stored beyond the NXmx standard. The complete file layout — master vs data files, the three format variants (NXmxLegacy, NXmxVDS, NXmxIntegrated), every NXmx field that is populated and every Jungfraujoch extension — is documented in HDF5 / NeXus data format.

If data collection was configured with a header_appendix containing a key hdf5 whose value is a JSON object of numbers and strings, those entries are written to /entry/user.

Other formats (CBF and TIFF)

Earlier versions could also write Crystallographic Binary File (CBF, miniCBF) and TIFF images. These writers have been removed: Jungfraujoch now writes only NXmx HDF5. The CBF and TIFF values are retained in the file-format enum for wire back-compatibility, but a request to write either format is rejected.

No file option(s)

There are two options to disable writing of files by the writer:

  • Setting file_prefix to empty string - this will disable sending files on ZeroMQ image socket.

  • Setting file format to NoFile - files are streamed over ZeroMQ socket, but jfjoch_writer will not write anything. This can be useful for debugging purposes, or if you only rely on republishing functionality of the jfjoch_writer

\ No newline at end of file diff --git a/LICENSE.html b/LICENSE.html new file mode 100644 index 00000000..8980d2db --- /dev/null +++ b/LICENSE.html @@ -0,0 +1,20 @@ + License — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

License

Jungfraujoch software is licensed with GPLv3 license. Jungfraujoch FPGA is licensed with CERN OHL-S license (see FPGA license).

GNU GENERAL PUBLIC LICENSE

Version 3, 29 June 2007

Copyright (C) 2007 Free Software Foundation, Inc. https://fsf.org/ Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.

Preamble

The GNU General Public License is a free, copyleft license for software and other kinds of works.

The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program–to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.

When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.

To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.

For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.

Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.

For the developers’ and authors’ protection, the GPL clearly explains that there is no warranty for this free software. For both users’ and authors’ sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.

Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users’ freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.

Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.

The precise terms and conditions for copying, distribution and modification follow.

TERMS AND CONDITIONS

  1. Definitions.

“This License” refers to version 3 of the GNU General Public License.

“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.

“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.

To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.

A “covered work” means either the unmodified Program or a work based on the Program.

To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.

To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.

An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.

  1. Source Code.

The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.

A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.

The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.

The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work’s System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.

The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.

The Corresponding Source for a work in source code form is that same work.

  1. Basic Permissions.

All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.

You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.

Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.

  1. Protecting Users’ Legal Rights From Anti-Circumvention Law.

No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.

When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work’s users, your or third parties’ legal rights to forbid circumvention of technological measures.

  1. Conveying Verbatim Copies.

You may convey verbatim copies of the Program’s source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.

You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.

  1. Conveying Modified Source Versions.

You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:

a) The work must carry prominent notices stating that you modified it, and giving a relevant date.

b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.

c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.

d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.

A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation’s users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.

  1. Conveying Non-Source Forms.

You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:

a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.

b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.

c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.

d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.

e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.

A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.

A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.

“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.

If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).

The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.

Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.

  1. Additional Terms.

“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.

When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.

Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:

a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or

b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or

c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or

d) Limiting the use for publicity purposes of names of licensors or authors of the material; or

e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or

f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.

All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.

If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.

Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.

  1. Termination.

You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).

However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.

Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.

Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.

  1. Acceptance Not Required for Having Copies.

You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.

  1. Automatic Licensing of Downstream Recipients.

Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.

An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party’s predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.

You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.

  1. Patents.

A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor’s “contributor version”.

A contributor’s “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.

Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor’s essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.

In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.

If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient’s use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.

If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.

A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.

Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.

  1. No Surrender of Others’ Freedom.

If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.

  1. Use with the GNU Affero General Public License.

Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.

  1. Revised Versions of this License.

The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.

Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.

If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy’s public statement of acceptance of a version permanently authorizes you to choose that version for the Program.

Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.

  1. Disclaimer of Warranty.

THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.

  1. Limitation of Liability.

IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

  1. Interpretation of Sections 15 and 16.

If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.

END OF TERMS AND CONDITIONS

How to Apply These Terms to Your New Programs

If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.

To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.

<one line to give the program's name and a brief idea of what it does.>
+Copyright (C) <year>  <name of author>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+

Also add information on how to contact you by electronic and paper mail.

If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:

<program>  Copyright (C) <year>  <name of author>
+This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+This is free software, and you are welcome to redistribute it
+under certain conditions; type `show c' for details.
+

The hypothetical commands show w and show c should show the appropriate parts of the General Public License. Of course, your program’s commands might be different; for a GUI interface, you would use an “about box”.

You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see https://www.gnu.org/licenses/.

The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read https://www.gnu.org/licenses/why-not-lgpl.html.

Jungfraujoch exceptions to GPL

As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL.

We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose of simulating FPGA design on CPU.

If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch API it may be distributed under terms of your choosing without being subject to GPL requirements.

\ No newline at end of file diff --git a/NAMING.html b/NAMING.html new file mode 100644 index 00000000..b3357d3a --- /dev/null +++ b/NAMING.html @@ -0,0 +1 @@ + Naming — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Naming

The software is Swiss, and so are its names: both halves of the system are named after places in the Alps that are, in one way or another, about moving a lot of something up a steep mountain as efficiently as possible — usually by train. Throughput, in other words.

Part

Name

What it does

Streaming / acquisition

Jungfraujoch

Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing.

Data processing

Rugnux

Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the rugnux tool).

Jungfraujoch

The Jungfraujoch is a high mountain col in the Bernese Alps, the saddle (Joch is German for “yoke” or “col”) between the peaks Jungfrau and Mönch, at 3,466 m. It is the site of the High Altitude Research Station Jungfraujoch, whose long-running atmospheric measurements are co-operated by the Paul Scherrer Institute — the same institute that develops this software and the JUNGFRAU detector.

The name is also a small piece of word-play. PSI’s JUNGFRAU detector and DECTRIS’s EIGER detector are both named after Bernese Alps peaks (the famous trio is Eiger, Mönch, Jungfrau). The Jungfraujoch — the pass between Jungfrau and Mönch — is where those two detector worlds meet.

And it fits the theme of the whole project: the Jungfraujoch is reached by the Jungfraubahn, whose terminus is the highest railway station in Europe (3,454 m, the “Top of Europe”). It is the closest you can get to that summit in a genuinely high-throughput way — by train, moving crowds up the mountain — which is exactly what the streaming side of this software does with detector frames.

Pronunciation (German): Jungfraujoch ≈ YUNG-frow-yokh. “Jung” as in young, “frau” rhymes with cow, and the final “joch” ends in the guttural ch of Scottish loch or German Bach — not a hard k.

Rugnux

Piz Rugnux is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern Switzerland. (Piz is the Romansh word for “peak”.) It rises above the Albula line of the Rhaetian Railway (Rhätische Bahn), part of the “Rhaetian Railway in the Albula / Bernina Landscapes” — a UNESCO World Heritage Site (Welterbe).

That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude in very little horizontal distance, it corkscrews through a series of helical (spiral) tunnels looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a steep mountain efficiently — the same idea the data-processing side of this software is built around: pushing a large volume of diffraction data through the analysis pipeline.

So the theme is consistent — Swiss mountains, trains, and throughput — while keeping the two subsystems clearly distinct: Jungfraujoch streams, Rugnux processes.

Pronunciation (Romansh): Piz Rugnux ≈ peets roo-NYOOKS. The “gn” is a soft palatal ñ, as in canyon or Italian gnocchi, not two separate sounds.

What is Romansh?

Romansh (Rumantsch) is the fourth national language of Switzerland, alongside German, French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the canton of Graubünden. It survives in several regional idioms, brought together in a standard form called Rumantsch Grischun. Naming the processing engine with a Romansh mountain is a small nod to the least-spoken but no-less-Swiss corner of the country.

\ No newline at end of file diff --git a/OPENAPI.html b/OPENAPI.html new file mode 100644 index 00000000..16a06252 --- /dev/null +++ b/OPENAPI.html @@ -0,0 +1,2 @@ + OpenAPI — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

OpenAPI

OpenAPI specs

See document with detailed OpenAPI specs.

Python client

Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. For convenience, we provide Python client as jfjoch-client PyPi package. To install the client you can use pip tool:

pip install jfjoch-client
+

See API reference from the OpenAPI generator.

\ No newline at end of file diff --git a/OPENAPI_SPECS.html b/OPENAPI_SPECS.html new file mode 100644 index 00000000..657f9820 --- /dev/null +++ b/OPENAPI_SPECS.html @@ -0,0 +1 @@ + OpenAPI specification — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

OpenAPI specification

See document with detailed OpenAPI specs generated with Redocly.

\ No newline at end of file diff --git a/PIXEL_MASK.html b/PIXEL_MASK.html new file mode 100644 index 00000000..063718d5 --- /dev/null +++ b/PIXEL_MASK.html @@ -0,0 +1,13 @@ + Pixel mask — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Pixel mask

Mask format

Jungfraujoch follows generally NXmx format format for pixel mask. Pixel mask is described as 32-bit unsigned integer array of size the same as the image. Conditions to mask pixel are described by setting a particular bit to one. This way it is possible to encode reason why pixel is included in the pixel mask, also for one pixel there can be multiple reasons encoded at the same time.

Bit values are set as follows:

Bit 0 - gap (pixel with no sensor)

Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn’t set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask)

Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization)

Bit 8 - user defined mask

Bit 9 - beam stop shadow (found by rugnux --detect-beam-stop, on by default; see rugnux). Unlike the other bits this one belongs to the run that found it, not to the dataset: rugnux clears it at the start of every run, so a mask read back from a file that carries one starts clear. The user mask (bit 8) is left alone.

Bit 30 - module edge (only for PSI systems)

Bit 31 - chip edge interpolated pixel (multipixel)

Custom user mask

Jungfraujoch allows to upload custom user mask. This happens in two steps. First create mask in TIFF format:

import numpy as np
+import tifffile as tiff
+
+# Create a 2068x2164 numpy array filled with zeros, with 32-bit unsigned integers
+array = np.zeros((2068, 2164), dtype=np.uint32)
+
+# Mark the pixel (300, 400) with the value 1
+array[300, 400] = 1
+
+# Save the array as a TIFF file
+tiff.imwrite('mask.tiff', array)
+

Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8).

Then upload the mask to Jungfraujoch server:

curl -v http://<jfjoch_broker http address>/config/user_mask.tiff -XPUT --data-binary @mask.tiff
+
\ No newline at end of file diff --git a/RELEASE_CONTENTS.html b/RELEASE_CONTENTS.html new file mode 100644 index 00000000..188f0548 --- /dev/null +++ b/RELEASE_CONTENTS.html @@ -0,0 +1 @@ + Release contents — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Release contents

This page describes what a Jungfraujoch release ships and what each artefact needs on the target machine — which CPU instruction set the binaries were compiled for, which CUDA toolkit they were built against, and which runtime libraries are bundled rather than expected from the host.

The artefacts in the table below are built and published by the continuous-integration pipeline (.gitea/workflows/build_and_test.yml) when a tag is pushed. For how to install and configure the result see Deployment; for the package-repository URLs see Linux package repositories.

Artefacts

Artefact

Distributed via

Contains

.rpm / .deb packages

package repositories

The full server stack: jfjoch (broker, frontend, FPGA and detector tools), jfjoch-writer, jfjoch-viewer (incl. the XDS plugin), jfjoch-driver-dkms

jfjoch_viewer-<version>-linux-cuda<major>.tgz, ...-linux-cpu.tgz

Gitea release page

Portable Linux viewer package: jfjoch_viewer, rugnux, jfjoch_extract_hkl, jfjoch_recompress and the license notices

jfjoch-viewer-<version>-win64-cuda<major>.exe, ...-win64-cpu.exe

Gitea release page

Windows installer with the same four programs, plus the Qt runtime

jfjoch-writer .rpm / .deb

Gitea release page

The writer alone, for a file-writing machine without the rest of the stack

libjfjoch_xds_plugin.so.<version>

Gitea release page

XDS HDF5 read plugin (built on RHEL 8); see Integration with MX software

jfjoch-client

PyPI and the Gitea PyPI index

Generated Python OpenAPI client

Documentation

Read the Docs and the gitea-pages branch

This documentation set

The FPGA firmware (.mcs) images are attached to the release as well. The firmware is stable and is carried from version to version, and is rebuilt with Vivado (see FPGA smartNIC) when it needs to change — so a card keeps its image across a software upgrade unless the release notes say otherwise.

CPU instruction set

The architecture flags live in the CI configuration rather than in CMakeLists.txt, so a site building from source picks its own (x86-64-v4 on an AVX-512 cluster, -march=native, or the plain baseline the compiler defaults to). The released binaries are compiled to a fixed floor:

Release

Flags

Minimum CPU

Linux (all packages, and the portable .tgz)

-march=x86-64-v3 -flto=auto

AVX2 + FMA + BMI2 — Intel Haswell (2013) / AMD Zen (2017) and newer

Windows installer

/arch:AVX

AVX — Intel Sandy Bridge (2011) / AMD Bulldozer and newer

The Windows floor is lower because MSVC has no spelling for the x86-64-v2 level; /arch:AVX is the nearest one and implies SSE4.1/4.2, which is what actually matters — without it Eigen has no vectorised round and falls back to a libm call per element. Link-time optimisation is applied on Linux only.

A binary will fault with an illegal instruction on a CPU below its floor. If you must run on older hardware, build from source without the flags.

Operating-system floor

The .rpm / .deb packages are built per distribution (RHEL/Rocky 8 and 9, Ubuntu 22.04 and 24.04) and are tied to it. The portable viewer .tgz is built on RHEL 8, the oldest supported distribution, so its glibc floor is low enough to run on any newer Linux — that is what it is for, and why it replaces the per-distro packaging of the viewer on the release page. The Windows installer is built and verified on Windows 11.

CUDA and non-CUDA builds

Every binary artefact is released in two variants, cuda<major> and cpu. The CUDA variant adds the GPU fast-feedback indexer (ffbidx), the GPU FFT indexer and GPU image processing; the CPU-only variant runs the same pipeline on the CPU with the FFTW indexer, at much lower throughput.

The CUDA toolkit used is the one on the corresponding build machine: CUDA 12 for the RHEL 8 packages, CUDA 13 for RHEL 9, Ubuntu and Windows. The major version is part of the artefact and repository name, so a download is self-identifying. Building from source needs CUDA 12.8 or newer.

A CUDA build does not require a CUDA machine. Of the CUDA components only cuFFT is linked dynamically — the CUDA runtime and the fast-feedback indexer are linked statically — and cuFFT itself has no link-time dependency on the NVIDIA driver library. Jungfraujoch asks how many CUDA devices are present at start-up and treats “none” (including “no driver installed”) as zero GPUs, falling back to the CPU path. So a CUDA build starts and runs correctly on a machine with no NVIDIA GPU at all, provided the cuFFT runtime can be loaded:

  • Portable .tgz and Windows installer — cuFFT is part of the distribution, shipped next to the executable (on Linux found through an $ORIGIN rpath). Nothing else is needed: no CUDA toolkit, and on a GPU machine only the NVIDIA driver.

  • .rpm / .deb — cuFFT comes from the distribution’s own CUDA packages, so that one dependency is managed centrally with the rest of CUDA. Install the cuFFT package alongside, or use the nocuda repositories on a machine where CUDA is not wanted.

The cuFFT runtime is large (the Windows DLL is ~256 MB), so the CUDA artefacts are correspondingly bigger than the CPU ones — the other reason for shipping both.

On a machine with an NVIDIA GPU, take the CUDA variant: only that one uses the GPU.

Windows installer

The Windows artefact covers jfjoch_viewer and the portable analysis CLIs only; the rest of Jungfraujoch (broker, receiver, FPGA host, detector control) is Linux-only.

The toolchain bounds of the released installer are:

  • Visual Studio 2026 with the C++ (MSVC) toolset. MSVC is not optional — CUDA on Windows builds through it — and it is what the release is compiled with.

  • CUDA Toolkit 13.3 for the cuda13 variant.

  • Qt 6.11 for MSVC (msvc2022_64), including Qt Charts.

  • Ninja as the generator; zlib and Eigen 3.4 supplied from a build prefix.

The installer is generated with NSIS and bundles the Qt runtime (via windeployqt) and, on the CUDA variant, the cuFFT DLL — so the end user installs neither Qt nor a CUDA toolkit. The two variants share an install directory and Start Menu group and replace each other (CUDA is a strict superset); they are told apart by the installer filename and the Add/Remove Programs entry:

Build

Installer file

Add/Remove Programs

CUDA (default)

jfjoch-viewer-<version>-win64-cuda<major>.exe

Jungfraujoch (CUDA)

CPU-only

jfjoch-viewer-<version>-win64-cpu.exe

Jungfraujoch (CPU)

To build the viewer yourself on Windows, see jfjoch_viewer ▸ Building from source on Windows.

Licenses

Every package variant carries the project license, the third-party manifest and the verbatim license texts of the bundled dependencies under share/doc/jfjoch. See Third-party software notices.

\ No newline at end of file diff --git a/REPOSITORIES.html b/REPOSITORIES.html new file mode 100644 index 00000000..2702abad --- /dev/null +++ b/REPOSITORIES.html @@ -0,0 +1,5 @@ + Linux package repositories — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Linux package repositories

For convenience, we are providing package repositories. With versions including and excluding CUDA linking. We recommend to install Jungfraujoch viewer from nocuda repository and remaining packages from cuda12/cuda13 repository.

The repository name encodes two choices: the slsDetectorPackage version the packages were built against (slsdet8 = 8.0.2, slsdet9 = 9.2.0 — it must match the detector firmware) and whether CUDA is linked in. What ends up inside each package, and what it needs on the target machine, is described in Release contents.

RHEL based systems

For RHEL systems we provide the following repositories:

RHEL version

slsDetectorPackage

CUDA

Repository file

8.x

8.0.2

12.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo

8.x

9.2.0

12.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12.repo

8.x

8.0.2

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo

9.x

8.0.2

13.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13.repo

9.x

9.2.0

13.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13.repo

9.x

8.0.2

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda.repo

To install the repository, run:

dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo
+

RPMs are signed by the Gitea package registry as they are uploaded. If your system cannot verify the signature, set gpgcheck=0 in the repository file or install with --nogpgcheck.

We provide the following packages in the repository:

  • jfjoch — broker, web frontend, FPGA and detector command-line tools

  • jfjoch-driver-dkms — PCIe kernel-module source, built by DKMS

  • jfjoch-writer — HDF5 writer service

  • jfjoch-viewer — desktop viewer, the offline analysis tools and the XDS plugin

Ubuntu based systems

For Ubuntu systems, we also provide the following repositories:

sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc
+echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+sudo apt update
+

$distribution uses Ubuntu names jammy (22.04) and noble (24.04). $component can be set to cuda13 and nocuda. Only slsDetectorPackage 8.0.2 is built for Ubuntu.

The same four packages as above are provided: jfjoch, jfjoch-driver-dkms, jfjoch-writer and jfjoch-viewer. Up to 1.0.0-rc.160 the first of them was misnamed jfjoch-jfjoch; the current package replaces it, so apt upgrade handles the rename.

Ubuntu packages are currently only going through a very limited testing.

\ No newline at end of file diff --git a/RUGNUX.html b/RUGNUX.html new file mode 100644 index 00000000..86d8f66e --- /dev/null +++ b/RUGNUX.html @@ -0,0 +1,21 @@ + rugnux — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

rugnux

rugnux is the offline crystallographic data-analysis tool of Jungfraujoch — the data-processing half of the system (see Naming for where the name comes from). It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the results to a _process.h5 file, plus reflection files (.mtz/.cif/.hkl) when merging is requested.

It runs the same analysis code as the online and interactive tools, just driven from the command line over a file rather than a live detector stream.

Note. rugnux is under very active development. This page describes the tool and its options at a high level; the authoritative, always-current list of options is the program’s own usage message — run rugnux with no arguments.

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

On screen; a processing job can write the same files as rugnux

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Use rugnux to re-analyse data after acquisition, to experiment with processing parameters, or to produce merged intensities for downstream structure solution.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. The CUDA build provides the GPU fast-feedback indexer (ffbidx) and the GPU FFT indexer (fft); without CUDA only the CPU fftw indexer is available. Spot finding, integration and scaling run on the CPU and scale with the thread count (-N).

Input and output

Input is a single Jungfraujoch HDF5 master file (NXmx-based). Spots are always found by rugnux itself, including for the two-pass rotation first pass — the spot lists a dataset may already carry were found online, at the acquisition’s threshold and with its ice-band spots already discarded, so reusing them would hide the spot-finding settings from the lattice search.

Output (controlled by -o, --output-prefix, default output):

  • <prefix>_process.h5 — NXmx-compliant HDF5 with derived metadata (spots, indexing, integration, azimuthal integration, per-image statistics). See HDF5 / NeXus data format for the layout. Written by default only when not merging (i.e. under --no-merge); add --write-process-h5 to also write it when merging.

  • Merging is on by default (--no-merge disables it). The merged reflections are written in three formats — each has its uses downstream:

    • <prefix>.mtz — CCP4 MTZ (IMEAN/I(+)/I(-), French–Wilson F, FreeR_flag) for the CCP4 / phenix reflection tools.

    • <prefix>.cif — mmCIF, for deposition and as the self-describing native format (also carries the merging statistics, ISa, twinning and radiation-damage indicators).

    • <prefix>.hkl — SHELX HKLF 4 text (h k l I σ(I), fixed 3I4,2F8.2), the direct input for SHELXC / ANODE / SHELXD. Bijvoet mates are written separately (I(+) at +hkl, I(-) at -hkl) so the anomalous signal is preserved; intensities are put on a common scale so the largest value fits the fixed-width field (the absolute scale is irrelevant to SHELXC/ANODE), and the file ends with the 0 0 0 terminator record.

    All three carry the refined unit cell (from rotation indexing) and the space group determined from systematic absences (constrained to the indexed lattice symmetry). No-reference scaling additionally emits per-iteration <prefix>_iterN_scale.dat.

  • <prefix>_report.txt — the results report: what the run determined, in a form both a person and a beamline script can read. Always written, next to the files above. See The results report below.

Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. By default the written resolution is trimmed automatically where CC1/2 falls off (--resolution-cutoff cc-logistic, CC1/2 target 0.30); set --scaling-high-resolution to fix the limit by hand, or --resolution-cutoff off to keep the full range.

Reflection-file conventions

mmCIF. Standard items carry their standard meanings — _refln.intensity_meas / _intensity_sigma, the pdbx_I_plus/pdbx_I_minus and pdbx_F_plus/pdbx_F_minus anomalous pairs, _reflns.* and _reflns_shell.* for the merging statistics, _reflns.B_iso_Wilson_estimate for the Wilson B, and _cell.* / _diffrn_radiation_wavelength.wavelength for the geometry.

Anything rugnux reports that has no standard item is written under a jfjoch_ prefix, inside the standard category it belongs to. That is a deliberate choice: a reader that does not know these items ignores them, and one that does can find them without guessing.

item

meaning

_reflns.jfjoch_diffrn_ISa

Asymptotic I/σ in XDS’s sense: the whole-range 1/√(a·b) of the error model, so it can be read directly against a CORRECT.LP

_reflns.jfjoch_diffrn_ISa_asymptotic

The strong-reflection tier — the counting-subtracted scatter of well-measured groups. XDS has no equivalent, and it can only ever be the more optimistic of the two. Rotation path only

_reflns.jfjoch_error_model_a, _b

The error model in XDS’s convention, σ² = a(σ₀² + b·I²), so the ISa above is re-derivable from the file rather than taken on trust

_reflns.jfjoch_second_moment_I

Twinning second moment ⟨I²⟩/⟨I⟩² — 2.00 untwinned, 1.50 for a perfect twin

_reflns.jfjoch_L_test_mean_abs_L, _L_test_mean_L_squared

Padilla–Yeates L-test. ⟨|L|⟩ is 0.500 untwinned / 0.375 for a perfect twin; ⟨L²⟩ is 0.333 / 0.200. Written only when the test found pairs

_reflns.jfjoch_radiation_damage_relative_B

Relative B from the first to the last rotation batch (Ų); positive is the usual direction, high-resolution intensity fading with dose

_jfjoch_radiation_damage_batch.*

Per-batch loop: id, rotation_start_deg, relative_B

_diffrn_detector.jfjoch_distance_mm, _jfjoch_beam_center_x_pxl, _jfjoch_beam_center_y_pxl

The refined detector geometry actually used, which is not otherwise recoverable from the reflection file

Compatibility note. Before rc.161, _reflns.jfjoch_diffrn_ISa carried the asymptote, not the whole-range value. There is no version marker inside the file, so a number taken from an older .cif is not comparable with one taken from a newer one.

SHELX HKLF 4 (<prefix>.hkl). Fixed-format 3I4,2F8.2 — h k l I σ(I), one record per reflection, terminated by a 0 0 0 record — which is what SHELXC, SHELXD and ANODE expect. Two properties worth knowing before using it:

  • Bijvoet mates are written separately, I(+) at +hkl and I(-) at -hkl, so the anomalous differences survive into SHELXC; a reflection with no anomalous split is written once, as its mean.

  • Intensities are rescaled by a single global factor so the largest value fits the F8.2 field. I and σ(I) share that factor, so every ratio — and therefore the anomalous signal — is untouched, but the absolute scale is not meaningful. This matters only if you intend to compare magnitudes with another file; SHELXC and ANODE use ratios alone.

The results report

<prefix>_report.txt records what the run determined, next to the reflection files. It is written on every --mode mx and --mode scale run that has an output prefix — there is no option to enable or disable it. Two cases follow from that:

  • An empty output prefix (-o "", the “compute the statistics, persist nothing” mode) writes nothing, the report included.

  • --no-merge still writes a report. It determined an indexing and a geometry result, and those are recorded; the merging section then says MERGE= NOT_PERFORMED rather than being omitted, so the absence is a statement and not something a reader has to infer.

The report is never allowed to fail a run: if it cannot be written (unwritable path, full disk) the failure is logged as a warning and the run finishes normally.

Format

The model is XDS’s CORRECT.LP: prose and tables a crystallographer reads top to bottom, with a structure a script can consume without parsing prose.

  • KEY= value assignment lines. Every number worth extracting is one, so a consumer gets it with a single grep '^ISA= ' and never has to read a sentence. Key names are stable.

  • Fixed-width tables with a stable header row for anything that is genuinely tabular — the resolution shells, the space-group candidates, the sweep-quality ranges.

  • WARNING: lines, one per finding, in plain English: WARNING: Frames 500-600 out of beam (10.1 deg, scale 0.12 and CC 0.30 of the run, 2% scaled). grep '^WARNING:' finds every one.

  • Section banners (***…*** around a numbered title) delimiting the blocks.

REPORT_VERSION= is the format’s own version. Key names, table columns and the reason vocabulary below are an interface other software may depend on: they do not change without that number moving.

Sections, in order: 1. DATA SET, 2. INDEXING, 3. GEOMETRY POST-REFINEMENT (rotation only), 4. SPACE GROUP DETERMINATION, 5. SCALING AND MERGING, 6. TWINNING, 7. RADIATION DAMAGE, 8. SWEEP QUALITY, 9. WARNINGS.

Which pass. A rotation run integrates twice — once at the geometry in the input file (<prefix>_01.*), then again at the post-refined geometry (<prefix>.*) — and can integrate a third time if a guard rejects the second pass. There is one report, for the pass that became the canonical output, and PASS= / PASS_DECISION= in section 1 say which pass that is and on what evidence, so no number in the file is ambiguous about which geometry produced it.

Not in the report: timing, frame rates, thread counts, per-image progress and library banners. Those are process, not result, and stay on stdout.

Sweep quality and the reason vocabulary

Section 8 lists the stretches of the sweep over which the crystal delivered much less than the rest of the run — the feedback a beamline control system needs to tell an operator that a crystal should be recentred or recollected. Nothing is excluded on the strength of it; the frames still carry signal, and this is a message for the beamline, not a filter.

SWEEP_QUALITY_STATUS= COMPUTED
+SWEEP_QUALITY_COUNT= 1
+SWEEP_QUALITY_REASONS= no_diffraction crystal_out_of_beam weak_diffraction loss_of_centring radiation_damage
+SWEEP_ROTATION= 360.0
+FLUX_PEAK_TO_TROUGH= 1.03
+SCALE_MODULATION_PEAK_TO_TROUGH= 1.00
+
+  FIRST_IMAGE   LAST_IMAGE   N_IMAGES  ROTATION  REASON                SEVERITY   SCALE      CC   INDEXED
+  -----------  -----------  ---------  --------  --------------------  --------  ------  ------  --------
+          500          600        101      10.1  crystal_out_of_beam       0.83    0.12    0.30      0.02
+  -----------  -----------  ---------  --------  --------------------  --------  ------  ------  --------
+

SWEEP_QUALITY_STATUS distinguishes COMPUTED (the diagnostic ran; a count of 0 means the sweep was clean throughout) from NOT_COMPUTED (it did not run — no scaling and merging, or stills data). A consumer must not read a missing table or a zero count as “clean” without checking it. SWEEP_QUALITY_REASONS lists the whole vocabulary this version can emit, so an unknown code is distinguishable from a missing one.

Reason code

Meaning

no_diffraction

The range recorded essentially no diffraction from the indexed lattice.

crystal_out_of_beam

Frames were lost: over the range a per-image scale could be fitted far less often than over the run.

weak_diffraction

The frames all still index, but with much less intensity — the cause was not determined.

loss_of_centring

One cycle of modulation per revolution: the crystal is off the rotation axis.

radiation_damage

The range runs to the end of a sweep whose quality was already decaying.

The vocabulary is closed and stable: a code is never renamed, and never reused for a different meaning. New codes are only ever added, and adding one moves REPORT_VERSION.

The columns are: FIRST_IMAGE/LAST_IMAGE — inclusive, in processed-image ordinals (the numbering of <prefix>_image.dat and of every other per-image array rugnux writes; with -s/--stride the source image is start + ordinal * stride); ROTATION — the width of the range in degrees; SEVERITY — the fraction of the run’s typical diffracting power missing over the range, 0 (as good as the run) to 1 (nothing at all); SCALE and CC — the range’s mean per-image scale and CC-to-merge relative to the run median; INDEXED — the fraction of the range’s frames that were scaled at all. Every range also appears as a WARNING: sentence in section 9.

The same finding is written per image into the _process.h5 as /entry/MX/sweepQuality, when one is written — see HDF5.

Validating against a model (rugnux --model)

Given a PDB atomic model of the same structure, --model model.pdb scales the model structure factors to the merged amplitudes — fitting a flat bulk-solvent contribution and an overall anisotropic B — and reports R-work / R-free and the mean 2Fo-Fc density at the atom centres. It also writes <prefix>_2fofc.ccp4, <prefix>_fofc.ccp4 and <prefix>_maps.mtz next to the merged reflections. Nothing about the model is refined; it is only re-fractionalized into the data cell, so a deposited model with a slightly different cell still lines up.

It is a data-quality lens, independent of the internal statistics: R-free measures the merged intensities against external truth, where CC1/2 and Rmeas only measure them against themselves. It also settles the two things merged intensities alone cannot: the enantiomorph (data merged in P41212 against a P43212 model are reindexed into the model’s hand), and — when no reference MTZ has already fixed it — a merohedral indexing ambiguity, by keeping the candidate reindexing with the lowest R-free.

Re-scaling and re-merging (rugnux --mode scale)

The scale mode re-scales and merges the already-integrated reflections stored in a _process.h5 file, without re-running spot finding or integration. Use it to re-merge quickly with a different space group, resolution limit, anomalous setting or reference MTZ. It reuses the same -o/-N/-s/-e/-S/-A/-B/-z/--scaling-* options as the full run, and (unlike the full pipeline) does not run a space-group search, so pass -S for the correct symmetry.

Detector calibration from powder rings (rugnux --mode calibration)

The calibration mode determines the detector geometry — PONI x/y, the two tilts rot1/rot2 and the distance — from the powder rings of a calibrant, and writes it as a pyFAI <prefix>.poni file alongside a printed report of how far each parameter moved from the header. Bragg data pin the beam centre worst (it is gauge-coupled to the crystal orientation); a powder ring has no orientation to be coupled to, so this is the measurement that fixes it.

rugnux --mode calibration --calibrant lab6 -N 8 -o det LaB6_master.h5
+

--calibrant takes lab6, agbh (silver behenate), ceo2, si or ice, case-insensitively. ice calibrates a real experiment against its own ice rings — no calibrant exposure needed — and is the reason a calibrant is a list of ring positions rather than a unit cell: hexagonal ice is P63/mmc, so rings enumerated from its cell would include systematically absent ones.

--calibration picks how the rings are measured, and both use the whole dataset — -s/-e/-t select which images:

  • rings (default) sums the (q × azimuth) azimuthal profile over every processed image into one map and fits the ring arcs in it. A powder ring is an arc, not a set of spots, and the summed profile measures it at every azimuth with all the run’s counts behind it. It needs the profile to be binned in azimuth, so this mode defaults --azim-phi-bins to 32.

  • spots pools the found spots of every processed image and fits those. It determines the centre from scratch (a Hough circle vote, which quantises it to a whole pixel) and then refines.

Both routes read the ring position out of a binned profile or a spot centroid, so the radial sampling matters: at a long detector distance the default 0.01 Å⁻¹ q bin is several pixels wide and quantises the rings route accordingly — pass a finer --azim-q-spacing there (the total q × azimuth bin count must stay under 65534).

The report prints the fitted geometry, the scatter of the ring points about the fitted rings and the standard error that implies on the centre. That error is formal: it measures the scatter of the points, not whether the rings themselves are trustworthy, so it stays small when a fit goes wrong for a structural reason — one visible ring, or ice that is textured rather than smooth.

Both the PONI (the point of normal incidence, which is what a .poni file stores) and the direct beam (where the beam lands, which is what most other programs call the beam centre) are printed. They differ by distance × tan(rot) once the detector is tilted, which on a 0.3° tilt at 300 mm is several pixels — enough to look like a disagreement with another program when there is none.

Quick start

Rotation data

Index, integrate, scale and merge a rotation sweep, fully de novo:

rugnux rotation_master.h5 \
+    -o rotation_run -N 32 \
+    --scaling-high-resolution 1.4
+

Because the dataset carries a rotation goniometer axis, it is processed as rotation data by default: two-pass rotation indexing (index the sweep once, then process every frame against that lattice) with the rot3d partiality model (rotation partials combined into 3D fulls). Scaling and merging run by default (for both rotation and stills; --no-merge turns them off); the unit cell is taken from the rotation indexer and the space group is determined from systematic absences, and both are written into the merged .cif.

Run fully de novo (no -C/-S) for the best result — supplying a cell or space group up front tends to degrade low-symmetry cases. --scaling-high-resolution (set it to your expected resolution) sharpens both the space-group search and the error model. To tune the first pass use --two-pass-rotation=100 (or -R100 — the first-pass image count); to force the sweep to be treated as independent stills use --force-still.

By default a rotation run also post-refines the geometry in a second pass: the first pass integrates and merges at the header geometry, then the detector distance + beam centre and the crystal cell / rotation-axis are refined against the merged fulls (cross-validated, and committed only for a small < 1 % move, with the gauge-weak beam centre restrained toward the header), and the second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical <prefix>_* output; the header-geometry pass is kept alongside as <prefix>_01_* for comparison. Disable it with --rotation-no-postrefine.

After the per-frame scale-fulls step, rotation scaling applies three correction surfaces, on by default (--no-scaling-corrections disables all):

  • Decay — a global Debye–Waller relative-B over the run, for the radiation damage that weakens later frames more at high resolution (a resolution×time systematic the resolution-flat per-frame scale cannot remove). It only engages when the total relative-B exceeds a physical floor (2 Ų). An optional --relative-b[=deg] extends this single global rate to a smooth per-batch relative-B curve (default 10°-of-rotation batches when bare, off otherwise), cross-validated like the surfaces here, for crystals whose decay is non-linear in dose.

  • Absorption — a smooth multiplicative factor over the diffracted-beam direction in the goniometer frame (path length through the crystal). Negligible at hard X-rays / thin crystals; it matters at low photon energy. Its benefit shows up most on model-based metrics: a smooth absorption error largely cancels among symmetry mates (little effect on the error model / ISa) but still biases the intensities, so it measurably lowers Rfree.

  • Modulation — a smooth multiplicative factor over the position where a reflection lands on the detector (a flat-field: detector-response and geometric systematics that vary across the detector plane). Symmetry-equivalents of one reflection land at different detector positions as the crystal rotates, which over-determines the surface. Because it lives in the detector frame (not the rotation) the same correction concept applies to stills. This is the largest of the three on JUNGFRAU data — it lowers Rmeas by several to tens of percent on datasets that carry a detector systematic, while holding or improving CC1/2 and the anomalous signal.

All three are cross-validated — fitted on even-numbered frames and kept only if they improve the held-out odd-frame symmetry-equivalent agreement by a clear margin (and vice versa). The agreement is scored as a σ-independent, Rmeas-like fractional deviation, so a surface can never pass cross-validation by merely reshaping the sigmas; where the systematic is absent the surface is a no-op rather than a source of added noise, which is why they are safe to leave on.

Independently of any correction, a rotation run prints a radiation-damage report — the per-image scale correlation-to-merge and mosaicity versus dose, and the relative B-factor change over the run (first→last) together with a per-batch relative-B curve, also written to the merged mmCIF. It is a data-quality-vs-dose diagnostic and never alters the merged intensities. A batch whose data cannot support a measurement prints - instead of a value, and the first→last number is printed only where a straight line describes the curve — damage is progressive, so a curve that dips and recovers is a disturbance of the sweep, not dose, and the report says so and points at the sweep-quality section (RADIATION_DAMAGE_RELATIVE_B= NOT_A_TREND).

Still / serial data

A dataset with no goniometer axis (e.g. a serial grid scan) is processed as independent stills automatically — no flag needed. Known-cell indexing with the GPU fast-feedback indexer, then merge against a reference structure:

rugnux serial_master.h5 \
+    -o serial_run -N 32 \
+    -X ffbidx -C 79,79,38,90,90,90 -S 96 \
+    -z reference.mtz \
+    --scaling-high-resolution 1.8
+

ffbidx requires a known cell (-C) and is the indexer of choice for sparse serial stills. The self-calibrating spot finder is on by default for both workflows (--no-adaptive-spots turns it off), and for serial stills leave --min-pix-per-spot unset so it is chosen per image — across the still-target battery this combination raises the indexing rate and typically extends resolution over a fixed threshold and fixed min-pix, at equal or better CC½. (You can still pin a fixed threshold with --spot-sigma / --spot-threshold and a fixed min-pix with --min-pix-per-spot.) If a dataset does carry a goniometer axis but you want per-frame stills processing anyway, add --force-still.

Command-line options

General:

Option

Description

-o, --output-prefix <txt>

Output file prefix (default: output)

-N, --threads <num>

Number of worker threads (default: all hardware threads)

-s, --start-image <num>

First image to process (default: 0)

-e, --end-image <num>

Last image to process (default: all)

-t, --stride <num>

Process every n-th image (default: 1)

-v, --verbose

Verbose output

Mode — --mode <name> (default mx):

Value

Description

mx

Full analysis — spot finding, indexing, integration and merging

azint

Only azimuthal integration (no spot finding/indexing); writes <prefix>_process.h5

scale

Only re-scale/merge the already-integrated reflections in the input _process.h5 (no re-integration)

calibration

Determine the detector geometry from powder rings; writes <prefix>.poni

Calibration (--mode calibration):

Option

Description

--calibrant <name>

Powder standard: lab6 | agbh | ceo2 | si | ice (default lab6, case-insensitive)

--calibration <txt>

How the rings are measured: rings | spots (default rings; see above). rings defaults --azim-phi-bins to 32

Detector mask:

Option

Description

--detect-beam-stop[=N|off]

Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. On by default (60 images); =off disables. Reflections behind the stop are attenuated but not flagged, so they integrate low with a plausible sigma and no existing rejection catches them

Geometry:

Option

Description

--estimate-beam-center

Measure the direct beam before indexing, from the symmetry of the spots where the sweep reaches at least half a turn and from the radial background profile where it does not; the value in the file is kept where neither can measure it. Off by default

--no-fit-spindle

With the above, keep the rotation axis given in the file instead of fitting its skew about the beam

Spot finding:

Option

Description

--spot-sigma <num>

Noise sigma level for spot finding (default: 4.0)

--spot-threshold <num>

Photon-count threshold for spot finding (default: 10)

--adaptive-spots

Self-calibrating detection (default, stills and rotation alike): the strong-pixel threshold comes from each image’s own per-resolution-ring noise instead of the fixed --spot-threshold, so one setting adapts across datasets (no per-dataset --spot-threshold/--spot-sigma tuning)

--no-adaptive-spots

Turn adaptive detection off and use the fixed --spot-threshold / --spot-sigma finder

--spot-false-pixels <num>

Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies --adaptive-spots)

--spot-high-resolution <num>

High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills

--spot-low-resolution <num>

Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit)

--min-pix-per-spot <num>

Minimum connected strong pixels per spot. If omitted, min-pix is chosen per image (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead.

--max-spots <num>

Maximum spots kept per image (the strongest ones) and handed to indexing (default: 1000)

--detect-ice-rings[=on|off]

Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file’s detect_ice_rings, or — where the file carries no such key — on for rotation and off for stills

Azimuthal integration (the radial profile behind the per-image ice-ring score):

Option

Description

-q, --azim-q-spacing <num>

Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings)

--azim-min-q <num>

Minimum Q, 1/Å

--azim-max-q <num>

Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see

--azim-phi-bins <num>

Number of azimuthal (phi) bins (default: 1)

--polarization-correction <on|off>

Enable/disable the azimuthal polarization correction

--solid-angle-correction <on|off>

Enable/disable the azimuthal solid-angle correction

Indexing:

A dataset with a rotation goniometer axis is processed as rotation data (two-pass rotation indexing) by default; a dataset without one is processed as independent stills. --force-still overrides the former; the -R / --single-pass-rotation / --force-rotation-lattice flags request rotation explicitly and pick the pass or lattice.

Option

Description

--force-still

Treat a rotation (goniometer) dataset as independent stills instead of rotation

-X, --indexing-algorithm <txt>

FFBIDX | FFT | FFTW | Auto | None

-C, --unit-cell <cell>

Reference unit cell "a,b,c,alpha,beta,gamma" (required by ffbidx)

-S, --space-group <num|symbol>

Space group number (92) or Hermann-Mauguin symbol (P43212) — for indexing and scaling

-r, --refine <txt>

Geometry refinement: none | orientation | beam_and_lattice (default) | flex (try all three per image, keep whichever indexes the most spots; alias multi)

-R, --two-pass-rotation[=num]

Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100)

--single-pass-rotation[=num]

Online-like single-pass rotation indexing (optional min angular range, deg)

--force-rotation-lattice <vec>

Force rotation lattice (9 floats, Å), skipping the first pass

--rotation-no-postrefine

Rotation: disable the default-on two-pass geometry post-refine (see the rotation section)

--refine-geometry[=N|off]

Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (-C / -z), =off disables

--index-ice-rings[=on|off]

Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: off; no effect without --detect-ice-rings, which does the flagging)

Indexer choice in brief: ffbidx (GPU) refines toward a known cell and is best for sparse serial stills; fft (GPU) / fftw (CPU) index de novo and suit strong rotation data. See the CPU/GPU data-analysis reference for the algorithms.

Scaling and merging:

Option

Description

--no-merge

Skip scaling and merging (on by default); write only the per-image _process.h5

-A, --anomalous

Anomalous mode (keep Friedel pairs separate)

--scale-fulls / --no-scale-fulls

rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills

--smooth-g[=deg]

rot3d: smooth the per-frame scale G over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off)

--no-scaling-corrections

rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below)

--relative-b[=deg]

rot3d: fit a per-batch relative-B beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise)

--simple-stills

Stills: treat every reflection as a full (p = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement

--no-expected-variance-merge

Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation’s signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting

--capture-uncertainty <num>

rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise)

--min-captured-fraction <num>

rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off)

--scaling-high-resolution <num>

High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below)

--scaling-low-resolution <num>

Low-resolution limit for scaling and merging, Å (default: 50, the value XDS configurations use; 0 removes the limit). Reflections coarser than this sit behind or beside the beam stop and are measured on a background it has eaten into

--resolution-cutoff <txt>

Automatic high-resolution cutoff for the written reflections and reported shells: cc-logistic | off (default: cc-logistic; ignored when --scaling-high-resolution is set)

--resolution-cc-target <num>

CC1/2 target defining the cc-logistic fall-off (default: 0.30)

--resolution-shells <num>

Number of resolution shells in the reported statistics table (default: 10)

--min-partiality <num>

Minimum partiality to accept a reflection (default: 0.02)

--ice-min-score <num>

Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before any ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The eleven fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness

--ice-min-spot-ratio <num>

The second ice-presence channel: found spots on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so --ice-min-score alone is blind to it (default: 2.0; 0 disables this channel)

--reject-outliers <num>

Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rot3d, off otherwise)

--min-image-cc <num>

Per-image CC limit, percent (default: no limit)

--search-min-zeta <num>

De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry |ζ| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences

--mosaicity <num>

Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed

--scaling-iterations <num>

Scaling iterations with no reference data (default: 3)

-z, --reference-mtz <file>

Reference MTZ (enables reference-driven scaling)

--reference-column <label>

Reference MTZ column to use (default: auto — F-model, else IMEAN/I/…)

--model <file.pdb>

After merging, validate the merged intensities against this atomic model (see below)

--write-process-h5

Also write the (large) _process.h5 when merging (default: only .mtz/.cif)

Integration:

Option

Description

--integrator <txt>

Spot integrator: gaussian (profile-fit, default) | empirical | boxsum (classical fallback)

--integration-radius <r>

Signal-box radius r1, or r1,r2,r3 (px). One value ⇒ r2=r1+2, r3=r1+4

--integration-stencil <k>

Push the r2..r3 background ring out by k times the beam’s radial streak bandwidth·R_px, per reflection (default 0 = the fixed circular ring). A fixed ring otherwise ends up on a streaked reflection’s own tails at high resolution and measures them as background. Only the ring moves, and only radially — the r1 signal box stays a circle — and the growth is capped at 2·r3. The neighbour exclusion grows with it, so on a crowded pattern a few reflections can be left with too little background and dropped. Needs --bandwidth: on a monochromatic beam the streak is zero and this does nothing

--background-clip <n>

Monochromatic (rotation + still): high-side clip of the background ring at mean + n·√mean (default 4; 0 = off). The default background estimator — it rejects neighbour cores and zingers without the symmetric trim’s Poisson skew bias. Broadband data always clip, at 3σ; ignored by --integrator boxsum

--background-trim <f>

Use the old symmetric trimmed mean for the background ring instead of the clip, 0≤f<0.5 (0.10 was the former default). Switches --background-clip off. A symmetric trim is biased low on Poisson data and adds ~5 counts to every partial, so this is for back compatibility only; 0 = plain ring mean. Rings holding more than 512 pixels fall back to the plain mean (the GPU sorts the ring in shared memory and the CPU now matches it), which the default radii never reach but wide ones do

--background-radial[=on|off|auto]

Correct the background ring for the curvature of the radial background (default off). Disk and ring are concentric, so a background linear in position cancels between them and only curvature survives — which on a smooth ice ring reaches +26 counts on a single reflection. auto applies it per image where that image’s ice score shows a smooth powder ring, since the model is a function of radius alone: on ice made of discrete crystallite spots there is no smooth ring and the correction makes the bias worse. Ignored by --integrator boxsum (no clip pass to take the curve from)

--integration-high-resolution <num>

High-resolution limit for prediction and integration. Omitted (or 0) means integration extends as far as the detector reaches — which is what the predictor can place on the detector anyway, since it rejects reflections that miss it. Set a value to integrate less than the detector offers

--max-hkl <n>

Predict reflections with |h|,|k|,|l| ≤ n (max 511). By default this is derived per crystal from the refined cell as ceil(max(a,b,c)/d_min) + 1, which is the exact bound: the predictor keeps only |q| ≤ 1/d_min and h = a·q, so no reflection can lie outside it and no candidate inside it is wasted on a shorter axis. Set it only to override that

--bandwidth <num>

Relative X-ray bandwidth FWHM (e.g. 0.01 for a 1% DMM); default from file or 0 (monochromatic)

Geometry overrides (defaults are taken from the input file; override them to reprocess with a corrected geometry):

Option

Description

--beam-x <num>

Beam centre X (pixel)

--beam-y <num>

Beam centre Y (pixel)

--detector-distance <num>

Detector distance (mm)

--wavelength <num>

Wavelength (Å)

--rot1 <num>

PONI detector rotation 1 (rad)

--rot2 <num>

PONI detector rotation 2 (rad)

--polarization <num>

Polarization factor

\ No newline at end of file diff --git a/SOFTWARE.html b/SOFTWARE.html new file mode 100644 index 00000000..243ff78a --- /dev/null +++ b/SOFTWARE.html @@ -0,0 +1 @@ + Software requirements — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Software requirements

Operating system

Recommended operating system is Red Hat Enterprise Linux (RHEL) / Rocky Linux versions 8 or 9. For this operating systems we provide RPMs with pre-built binaries to simplify deployment. On experimental basis we also build repositories for Ubuntu 22.04 and 24.04.

Running Jungfraujoch on Red Hat Enterprise Linux 7 is currently not tested and not recommended, but likely possible with providing some packages from external repositories.

The desktop viewer jfjoch_viewer (only) additionally runs on Windows 11, where it is shipped as a pre-built installer; it can also be built from source with Visual Studio 2026 (MSVC), CUDA 13.3 and Qt 6.11 — see jfjoch_viewer ▸ Building from source on Windows. The Windows installer bundles the Qt runtime, and on the CUDA build the CUDA runtime (cuFFT) as well, so end users need neither Qt nor a CUDA toolkit installed — only an NVIDIA GPU driver for the GPU path. The rest of Jungfraujoch is Linux-only. See Release contents for the CPU baseline and CUDA requirements of each released package.

Software dependencies

Required:

  • C++20 compiler and C++20 standard library; recommended GCC 11+ or clang 14+ (Intel OneAPI, AMD AOCC)

  • CMake version 3.26 or newer + a build tool (GNU make or Ninja)

  • zlib compression library

  • Eigen (header-only linear algebra library), version 3.4.x (the build requests Eigen3 3.4, which Eigen’s same-major-version rule does not satisfy with 5.x)

HDF5, libtiff and libjpeg-turbo used to be required system packages; they are now downloaded and built automatically by CMake (see the note below), so they no longer need to be installed.

Optional:

  • CUDA compiler version 12.8 or newer - required for the MX fast feedback indexer and GPU analysis

  • FFTW library - for indexing if GPU/CUDA is absent (also auto-downloaded by CMake)

  • Node.js - to build the frontend

  • Qt version 6 (for jfjoch_viewer)

Many further dependencies (spdlog, Zstandard, HDF5, slsDetectorPackage, libzmq, libtiff, libjpeg-turbo, Ceres, the fast feedback indexer, Catch2, …) are downloaded automatically by CMake and statically linked; building therefore requires network access on the first configure. zlib and Eigen are the exception — they must be preinstalled (found via find_package); on Windows, where they are not present system-wide, install them into a prefix and point CMAKE_PREFIX_PATH at it (see Building from source on Windows). Others are vendored directly in the source tree. The complete list of third-party components, with copyright holders, licenses and verbatim license texts, is in Third-party software notices and the licenses/ directory.

\ No newline at end of file diff --git a/SOFTWARE_INTEGRATION.html b/SOFTWARE_INTEGRATION.html new file mode 100644 index 00000000..2e1e57d9 --- /dev/null +++ b/SOFTWARE_INTEGRATION.html @@ -0,0 +1,2 @@ + Integration with MX data processing software — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Integration with MX data processing software

XDS

Jungfraujoch files are compatible with XDS, but there is a need of a dedicated plugin. First we recommend to use Jungfraujoch own XDS plugin. It is available for Linux only and can be downloaded from Gitea release directory (compiled on RHEL 8), it is also distributed in jfjoch_viewer RPM/APT packages. To use the plugin, download the file libjfjoch_xds_plugin.so.1.0.0 (three numbers at the end represent version of the plugin, and can differ later in time), save it to common directory (e.g., /opt/xds) and add the following line in the XDS.INP file:

LIB="/opt/xds/libjfjoch_xds_plugin.so.1.0.0"
+

We are also testing XDS with Durin and Neggia plugins, though they don’t have full functionality:

  • Neggia plugin doesn’t support HDF5 virtual data sets. It can be downloaded from github.com/dectris/neggia.

  • Durin has known bugs with handling non-DECTRIS files (so with virtual data sets or single format HDF5 file format). We recommend Durin plugin prepared by the Global Phasing consortium: github.com/CV-GPhL/durin, rather than original from the Diamond Light Source.

DIALS

Jungfraujoch files are tested regularly with DIALS (currently v. 3.27.0) xia2.ssx pipeline for serial crystallography. There is one known limitation: files generated with NXmxLegacy format (mimicking DECTRIS filewriter1 format) are not handled properly with DIALS. VDS based HDF5 format (NXmxVDS) is recommended, when using DIALS.

CrystFEL

Jungfraujoch files are compatible with CrystFEL.

\ No newline at end of file diff --git a/TESTS.html b/TESTS.html new file mode 100644 index 00000000..67815e34 --- /dev/null +++ b/TESTS.html @@ -0,0 +1,12 @@ + Tests — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Tests

The unit and integration tests are written with Catch2 and collected into a single binary, tests/jfjoch_test. Build and run it with:

make -j$(nproc) jfjoch_test
+cd tests
+./jfjoch_test                 # everything
+./jfjoch_test "<test name>"   # one test case
+./jfjoch_test "[tag]"         # by tag
+

There are also benchmark and hardware routines, each printing its own usage:

  • jfjoch_hdf5_test to measure HDF5 dataset writing speed (single threaded). It doubles as the generator of the HDF5 files used by the external-software tests below.

  • jfjoch_lite_perf_test to measure the CPU/GPU (“lite”) analysis path - indexing, integration and optional file writing.

  • jfjoch_fpga_test to test quality/performance of FPGA card(s) and software routines. With -H it runs the high-level-synthesis C model on the CPU, so no FPGA device is needed.

Out-of-space handling is covered separately by jfjoch_hdf5_enospc_test, run under the enospc_shim LD_PRELOAD module that makes writes fail with ENOSPC.

In addition, tests are executed to verify that datasets written by Jungfraujoch are readable by other MX software (see Integration with MX data processing software) - XDS through the Jungfraujoch, Durin and Neggia plugins, and DIALS xia2.ssx - for each of the NXmx layouts. Input files for these programs are placed in the tests/xds, tests/xds_durin, tests/xds_neggia and tests/crystfel folders. See .gitea/workflows/build_and_test.yml for the exact commands; the CrystFEL fixtures are run by hand rather than in the pipeline.

Judging a change to the analysis itself

Two harnesses in the repository root run rugnux over a directory of stored datasets and score the result. Neither is part of CI - run them when a change plausibly moves merged results, not as a reflex. Both take their dataset list from outside the repository, because dataset and sample identities are not committed.

  • rugnux_vs_xds.py - the rotation battery. Runs rugnux de novo over every crystal under a data root and tabulates reflections, observations, space group, R_meas, CC1/2, ISa and wall-clock time against the XDS CORRECT.LP beside each dataset.

  • rugnux_anomalous.py - the anomalous-peak-height arbiter, below.

The anomalous-peak-height arbiter

A change that touches partiality - a mosaicity estimator, a rocking-curve model, a background change, anything that alters how partial reflections are weighted - cannot be judged by the statistics we normally reach for:

statistic

why it fails for this class of change

ISa, R_meas, error-model b

one measurement, not three; dominated by the low-resolution shells; not invariant to the uniform intensity rescale a partiality change produces

last-shell R_meas

moves with its denominator, i.e. the wrong way by construction

rugnux --model R-free

tracks its own zero-information floor, which moves ~22x more than R-free itself over the same sweep

per-shell agreement with XDS_ASCII.HKL

XDS never divides by partiality, so “divide less” moves us toward it mechanically; measured to put the optimum ~1.4x too low

Anomalous difference density at known scatterer sites has none of these problems. It is read in units of the map’s own sigma, so a uniform intensity rescale cancels exactly, and it is referenced to the structure rather than to another program’s partiality model.

rugnux_anomalous.py measures it: shelxc + anode -a (CCP4) on each arm’s merged reflections, against a model that is placed once and then held fixed. It reports, per dataset, the mean site height and the off-site noise floor, and, between arms, the paired per-site change.

# compare two arms (each a directory of <id>/<id>.hkl + .mtz)
+./rugnux_anomalous.py --config <table>.json  base=<dir-A>  test=<dir-B>
+
+# a parameter scan: numeric labels turn the arms into a curve with a per-dataset optimum
+./rugnux_anomalous.py --config <table>.json \
+    0.85='<scan>/{name}/s0p85.hkl' 1.00='<scan>/{name}/s1.hkl' 1.20='<scan>/{name}/s1p2.hkl'
+

An arm is a rugnux output directory or a path template containing {name}. --place does the one-off model placement, --write-config-template prints the config skeleton, and ANODE results are cached under the config’s workdir (a full 9-dataset x 11-arm scan takes under a minute).

The gate. A dataset counts only if its reference arm shows top peak > 1.5x the highest off-site peak and at least 3 sites over 5 sigma. A dataset that fails is reported as EXCLUDED, never as a zero - the difference between two noise measurements is not a measurement.

Standing dataset set (2026-08): 8 datasets from 7 crystals, 114 sulfur sites, all judged on native sulfur signal.

crystals

space group

photon energy

sites each

2

P41212

12.4, 16.0 keV

18

2 (lysozyme)

P43212

13.0, 5.0 keV

27

3 (4 datasets - one crystal contributes two energies)

cubic, I-centred

13.0, 6.0, 5.0, 5.0 keV

6

Report n as crystals, not datasets: two energies of one crystal are not two independent votes, and the tool prints both counts for that reason.

Traps this tool exists to encapsulate. Every one of them has already cost a working day:

  1. The phasing space group comes from the config, never from the merged file. I23 and I213 have identical systematic absences (I-centring already forces the screw condition), so no data can separate them, and phaser’s automatic space-group test only tries the enantiomorph - which for I23 is itself. Phasing an I-centred cubic case in the I23 that both rugnux and XDS report gives TFZ 7-11 where the other member gives 30-50, and drops the mean site height by a factor 3-10 - enough to make four good datasets look signal-free. Thirteen classes of chiral space group are indistinguishable this way; --place tries every member of the class and reports each one’s LLG/TFZ.

  2. Place the model once, from a reference arm, and reuse it unchanged. Re-phasing per arm lets the model move and contaminates the comparison. Refining the placed model against the dataset’s own amplitudes is allowed (it lifts the peaks another 4-10%) as long as the same refined model is then used for every arm.

  3. The gate and the measurement must use the same model. Gating on one model and scoring the curve with another silently changes which datasets are in the set.

  4. The off-site floor skips special positions. A peak on the cell origin is a ripple of the calculated phases, not a sample of the background; leaving it in inflates the floor by several sigma and can turn a passing dataset into a failing one. Such peaks are reported in their own spec column rather than dropped silently.

Reading the result. Judge the paired per-site change, with its standard error, pooled over crystals. A per-dataset optimum whose arm does not beat the reference on the paired test is flagged not significant vs ref and must not be quoted as a preference; so must one sitting on the edge of the scanned grid (grid edge) - extend the grid instead.

\ No newline at end of file diff --git a/THIRD_PARTY_NOTICES.html b/THIRD_PARTY_NOTICES.html new file mode 100644 index 00000000..65a9fdc3 --- /dev/null +++ b/THIRD_PARTY_NOTICES.html @@ -0,0 +1,2 @@ + Third-party software notices — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Third-party software notices

Jungfraujoch is licensed under GPL-3.0 (see LICENSE); the FPGA design is licensed under CERN-OHL-S-2.0 (see fpga/LICENSE). It builds on a number of third-party components, acknowledged below as required by their licenses.

This file is the human-readable manifest. The verbatim license texts live in the licenses/ directory (regenerate with bash licenses/COLLECT.sh). The frontend’s bundled JavaScript dependencies are listed separately in frontend/dist/THIRD_PARTY_LICENSES.txt, generated at build time (npm run licenses).

All licenses below are GPL-3.0-compatible.

Fetched at build time and statically linked into the C++ binaries

These are downloaded by CMake (FetchContent / ExternalProject) during the first configure and linked into the Jungfraujoch executables.

Component

Version

Copyright

License (SPDX)

License text

spdlog

1.17.0

Gabi Melman

MIT

spdlog.txt

Zstandard

(pinned)

Meta Platforms, Inc.

BSD-3-Clause

zstd.txt

HDF5

2.1.0

The HDF Group; UIUC

BSD-3-Clause-style

hdf5.txt

slsDetectorPackage

8.0.2 / 9.2.0

PSI

LGPL-3.0-or-later

LGPL, GPL

cpp-httplib

0.39.0

Yuji Hirose

MIT

cpp-httplib.txt

libzmq (ZeroMQ)

4.3.5

iMatix and contributors

MPL-2.0

libzmq.txt

libtiff

4.7.1

Sam Leffler; SGI

libtiff (BSD-like)

libtiff.txt

FFTW

3.3.10

Matteo Frigo; MIT

GPL-2.0-or-later

fftw.txt

Ceres Solver

(pinned)

Google Inc. and contributors

BSD-3-Clause

ceres-solver.txt

fast-feedback-indexer

(pinned)

PSI

BSD-3-Clause

fast-feedback-indexer.txt

libjpeg-turbo

(pinned)

D. R. Commander and others; IJG

IJG + BSD-3-Clause + Zlib

libjpeg-turbo.txt

Catch2

3.13.0

Catch2 Authors

BSL-1.0

catch2.txt

Catch2 is used only to build the test binary (jfjoch_test) and is not part of any shipped artifact; it is listed here for completeness.

Vendored directly in the repository

These are copied into the source tree (see the path) rather than fetched.

Component

Path

Copyright

License (SPDX)

License text

nlohmann/json

include/nlohmann/

Niels Lohmann

MIT

nlohmann-json.txt

Macaron Base64

include/base64/

tomykaira

MIT

base64-macaron.txt

TinyCBOR

frame_serialize/tinycbor/

Intel Corporation

MIT

tinycbor.txt

Bitshuffle

compression/bitshuffle/

Kiyoshi Masui

MIT

bitshuffle.txt

Bitshuffle (h-perf)

compression/bitshuffle_hperf/

Kal Cutter (DECTRIS)

Apache-2.0

bitshuffle-hperf.txt

LZ4

compression/lz4/

Yann Collet

BSD-2-Clause

lz4.txt

HLS arbitrary-precision types

fpga/include/

Xilinx, Inc.

Apache-2.0

xilinx-hls-headers.txt

GEMMI

gemmi_gph/

Global Phasing Ltd.

MPL-2.0

gemmi.txt

traccc (ACTS)

image_analysis/spot_finding/StrongPixelSet.cpp, SpotExtractorGPU.cu

CERN, for the benefit of the ACTS project

MPL-2.0

traccc.txt

xbflash.qspi

tools/xbflash.qspi/

Xilinx / AMD

Apache-2.0

xbflash-qspi.txt

wingetopt

tools/wingetopt/

Todd C. Miller; The NetBSD Foundation

ISC AND BSD-2-Clause

wingetopt.txt

Runtime libraries and SDKs (shipped in binaries, not in the source tree)

Component

Used by

License

Notice

Qt 6

jfjoch_viewer

LGPL-3.0

notice, LGPL-3.0

NVIDIA CUDA Toolkit (cudart, cuFFT)

CUDA builds

NVIDIA CUDA EULA

notice, EULA

zlib

everywhere (compression)

Zlib

zlib.txt

Eigen

analysis libs, Ceres, ffbidx (header-only)

MPL-2.0 (+ BSD parts)

eigen.txt, README

Frontend (npm) dependencies

The React/TypeScript frontend (frontend/) bundles a large transitive tree of npm packages, overwhelmingly MIT/ISC/BSD/Apache-2.0 licensed. Their full notices are generated automatically:

cd frontend && npm run licenses     # writes dist/THIRD_PARTY_LICENSES.txt
+

The generated file is produced as part of the frontend build target and installed alongside the served frontend, so the shipped web UI carries its own attribution.

Notes on weak-copyleft and attribution-sensitive components

  • MPL-2.0 (Eigen, GEMMI, libzmq, traccc): file-level copyleft. GEMMI is vendored in gemmi_gph/ in trimmed form; libzmq is fetched at build time; Eigen is provided externally (header-only). The corresponding source is available from each project upstream.

  • traccc is the one entry that is not a vendored directory. Its sparse connected-component labelling enters two otherwise first-party files: StrongPixelSet.cpp adapts the SparseCCL source, and SpotExtractorGPU.cu follows the design of its GPU counterpart. MPL-2.0 is file-level, so both files name the origin at the top and are covered by licenses/traccc.txt. See ACKNOWLEDGEMENT.md for the citation.

  • FFTW is GPL-2.0-or-later — compatible with, and absorbed by, this project’s GPL-3.0 license.

  • Apache-2.0 components: where upstream ships a NOTICE file, it is reproduced in the corresponding licenses/ text.

  • Qt (LGPL-3.0) and NVIDIA CUDA (EULA) carry redistribution conditions beyond a copyright notice; see their dedicated notice files. The verbatim LGPL-3.0 and CUDA EULA texts are bundled (licenses/Qt6-LGPL-3.0.txt, licenses/NVIDIA-CUDA-EULA.txt); the CUDA EULA is the one shipped with CUDA Toolkit 12.8 — replace it if you build against a different toolkit version.

\ No newline at end of file diff --git a/TOOLS.html b/TOOLS.html new file mode 100644 index 00000000..275d45f3 --- /dev/null +++ b/TOOLS.html @@ -0,0 +1,13 @@ + Tools — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Tools

Besides the main services (jfjoch_broker, jfjoch_writer, jfjoch_viewer), the repository ships a number of command-line tools. Each prints its own usage when run with -h or without arguments.

Data analysis

rugnux

Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, integration, scaling/merging) on a stored HDF5 dataset, producing a _process.h5 file and, when merging, reflection files. Merging is on by default (--no-merge disables it). --mode picks what a run does: mx (the above, the default), azint (only azimuthal integration, no spot finding/indexing), scale (re-scale/merge the already-integrated reflections in a _process.h5 without re-integrating) or calibration (detector geometry from a calibrant’s powder rings, written as a .poni file). See rugnux.

jfjoch_extract_hkl

Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across neighbouring images and compare against an XDS INTEGRATE.HKL reference.

FPGA / PCIe card management

jfjoch_pcie_status

Prints detailed status information about the card. Safe to run during data collection:

./jfjoch_pcie_status /dev/jfjoch0
+

jfjoch_pcie_net_cfg

Reads and modifies the network configuration of the card’s interfaces:

jfjoch_pcie_net_cfg <device name>
+     Read configuration for all network interfaces of a device
+jfjoch_pcie_net_cfg <device name> <if number>|fgen
+     Read configuration for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen ipv4 <IPv4 address>
+     Set IPv4 address for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen direct 0|1
+     Set direct mode for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen clear
+     Clear Ethernet counters for a particular network interface / internal frame generator
+

jfjoch_pcie_clear_net_counters

Resets the card’s Ethernet, UDP and ICMP packet counters (which otherwise run from power-on):

./jfjoch_pcie_clear_net_counters /dev/jfjoch0
+

Testing, benchmarking and simulation

jfjoch_udp_simulator

UDP packet simulator used to test the Jungfraujoch FPGA receiver.

jfjoch_fpga_test

Exercises and benchmarks the FPGA data path and receiver. With -H it runs the high-level synthesis C model on the CPU, so no FPGA device is required.

jfjoch_lite_perf_test

Performance test of the lite (CPU/GPU) analysis path — indexing, integration and optional file writing.

jfjoch_hdf5_test

Tests single-threaded HDF5 writer performance.

jfjoch_simplon_test

Minimal test client for a DECTRIS SIMPLON detector API.

\ No newline at end of file diff --git a/VERSIONING.html b/VERSIONING.html new file mode 100644 index 00000000..db22233b --- /dev/null +++ b/VERSIONING.html @@ -0,0 +1 @@ + Semantic versioning — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Semantic versioning

Jungfraujoch is following semantic versioning. For this purpose we define public API as following:

  • OpenAPI configuration interface

  • CBOR serialization ZeroMQ stream

  • HDF5 file format

This means that changes in the format of thereof must be accompanied by version change - major version in case of breaking changes, minor version in case of feature expansion.

NOTE: FPGA design, PCIe driver, and internal libraries are not part of the public API and are considered internals of Jungfraujoch. Breaking changes in these components can happen without incrementing major version of the whole package. It will be marked in changelog.

\ No newline at end of file diff --git a/WEB_FRONTEND.html b/WEB_FRONTEND.html new file mode 100644 index 00000000..1ce08c62 --- /dev/null +++ b/WEB_FRONTEND.html @@ -0,0 +1 @@ + Web frontend — Jungfraujoch 1.0.0-rc.161 documentation Skip to content

Web frontend

Jungfraujoch is equipped with React-based web frontend for user-friendly experience. Frontend has the following options:

  • Presenting current state of the detector

  • Plotting results of online quality calculations

  • Showing live view images from the detector

  • JUNGFRAU calibration numbers

  • Configuring the detector, as well as pedestal/initialization operations

Frontend is written in TypeScript. For details see frontend/ directory.

\ No newline at end of file diff --git a/_images/jfjoch.png b/_images/jfjoch.png new file mode 100644 index 00000000..aa75a7ea Binary files /dev/null and b/_images/jfjoch.png differ diff --git a/_sources/ACKNOWLEDGEMENT.md.txt b/_sources/ACKNOWLEDGEMENT.md.txt new file mode 100644 index 00000000..27e47e7f --- /dev/null +++ b/_sources/ACKNOWLEDGEMENT.md.txt @@ -0,0 +1,113 @@ +# Acknowledgements + +Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). + +The project is supported by : +* Innosuisse via Innovation Project "NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications" (101.535.1 IP-ENG; Apr 2023 - Sep 2025). +* ETH Domain via Open Research Data Contribute project (Jan - Dec 2023) +* AMD University Program with donation of licenses of Ethernet IP cores and Vivado software + +Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading +the result, follows Jon Wright (ESRF): "Experiences with GPU decompression for bitshuffle + LZ4 +data", HDF5 User Group meeting (2021), and [bslz4decoders](https://github.com/jonwright/bslz4decoders). +The CUDA kernels in Jungfraujoch are its own, but the approach is his. + +Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the +ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, +A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP +experiments" (2025), [arXiv:2505.22822](https://arxiv.org/abs/2505.22822); +[traccc](https://github.com/acts-project/traccc). The CPU spot extractor adapts its SparseCCL source, +and the CUDA spot extractor follows the design of its GPU counterpart - a backward-neighbour graph +over a sorted hit list, resolved by a parallel union-find. traccc is MPL-2.0; see +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). + +This software uses Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license + +## Crystallographic methods adopted from other packages + +The analysis pipeline reimplements methods first published, and in most cases first implemented, by +other crystallographic software. The code below is Jungfraujoch's own; the methods are theirs, and +are acknowledged here. Where a package's source was consulted this is said explicitly. None of these +packages is linked or vendored, with the single exception of GEMMI (see +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)). + +**[XDS](https://xds.mr.mpg.de/)** — rotation geometry and notation, the reciprocal Lorentz and +partiality treatment, the maximum-likelihood mosaicity estimate, the `MINPK` criterion for rejecting +a reflection whose predicted profile is not cleanly its own, the intensity-based test for a +centred lattice, and the scaling correction surfaces indexed by image number and detector region. W. Kabsch, "XDS" (2010), Acta Cryst. D66, 125-132 +[doi:10.1107/S0907444909047337](https://doi.org/10.1107/S0907444909047337); W. Kabsch, "Integration, +scaling, space-group assignment and post-refinement" (2010), Acta Cryst. D66, 133-144 +[doi:10.1107/S0907444909047374](https://doi.org/10.1107/S0907444909047374). + +**Profile fitting** with reweighted, de-biased variances is the Kabsch/Otwinowski iteration, from the +second XDS paper above and from Z. Otwinowski and W. Minor, "Processing of X-ray diffraction data +collected in oscillation mode" (1997), Methods Enzymol. 276, 307-326 +[doi:10.1016/S0076-6879(97)76066-X](https://doi.org/10.1016/S0076-6879%2897%2976066-X). + +**[DIALS](https://dials.github.io/)** — the resolution cutoff from the CC1/2 fall-off, per-observation +outlier rejection at merge, the scaling error model, and the treatment of a reflection whose +background is contaminated. Its published behaviour, and in places its source, settled several +choices here. G. Winter, D. G. Waterman, J. M. Parkhurst et al., "DIALS: implementation and +evaluation of a new integration package" (2018), Acta Cryst. D74, 85-97 +[doi:10.1107/S2059798317017235](https://doi.org/10.1107/S2059798317017235); D. G. Waterman, +G. Winter, R. J. Gildea et al., "Diffraction-geometry refinement in the DIALS framework" (2016), +Acta Cryst. D72, 558-575 [doi:10.1107/S2059798316002187](https://doi.org/10.1107/S2059798316002187); +J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in the DIALS software +package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399 +[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198); J. M. Parkhurst, +G. Winter, D. G. Waterman et al., "Robust background modelling in DIALS" (2016), J. Appl. Cryst. 49, +1912-1921 [doi:10.1107/S1600576716013595](https://doi.org/10.1107/S1600576716013595). + +**[POINTLESS](https://www.ccp4.ac.uk/)** (CCP4) — the space-group search. Stage A scores each +candidate rotation operator by the correlation of I(h) with I(Rh); the screw-axis test scores a +predicted-absent class against the rest of its own axial row rather than against a global mean or a +fixed cut, and lets confidence fall away with the number of axial reflections instead of refusing +below a count. P. Evans, "Scaling and assessment of data quality" (2006), Acta Cryst. D62, 72-82 +[doi:10.1107/S0907444905036693](https://doi.org/10.1107/S0907444905036693); P. R. Evans, "An +introduction to data reduction: space-group determination, scaling and intensity statistics" (2011), +Acta Cryst. D67, 282-292 [doi:10.1107/S090744491003982X](https://doi.org/10.1107/S090744491003982X); +P. R. Evans and G. N. Murshudov, "How good are my data and what is the resolution?" (2013), Acta +Cryst. D69, 1204-1214 [doi:10.1107/S0907444913000061](https://doi.org/10.1107/S0907444913000061); +J. Agirre, M. Atanasova, H. Bagdonas et al., "The CCP4 suite: integrative software for macromolecular +crystallography" (2023), Acta Cryst. D79, 449-461 +[doi:10.1107/S2059798323003595](https://doi.org/10.1107/S2059798323003595). + +**[MOSFLM](https://www.mrc-lmb.cam.ac.uk/mosflm/)** — the Rossmann FFT autoindexing algorithm and +post-refinement practice, including which parameters are safe to refine per image and which must be +refined over a wedge. A. G. W. Leslie and H. R. Powell, "Processing diffraction data with MOSFLM" +(2007), in *Evolving Methods for Macromolecular Crystallography*, NATO Science Series II, vol. 245, +41-51 [doi:10.1007/978-1-4020-6316-9_4](https://doi.org/10.1007/978-1-4020-6316-9_4); +T. G. G. Battye, L. Kontogiannis, O. Johnson, H. R. Powell and A. G. W. Leslie, "iMOSFLM: a new +graphical interface for diffraction-image processing with MOSFLM" (2011), Acta Cryst. D67, 271-281 +[doi:10.1107/S0907444910048675](https://doi.org/10.1107/S0907444910048675); H. R. Powell, +T. G. G. Battye, L. Kontogiannis, O. Johnson and A. G. W. Leslie, "Integrating macromolecular X-ray +diffraction data with the graphical user interface iMosflm" (2017), Nat. Protoc. 12, 1310-1325 +[doi:10.1038/nprot.2017.037](https://doi.org/10.1038/nprot.2017.037). + +**[CrystFEL](https://www.desy.de/~twhite/crystfel/)** — spot finding, the three-ring integration +region, the serial/stills processing model, and the per-frame indexing acceptance test +(`indexing_peak_check()` in `peaks.c`). T. A. White, R. A. Kirian, A. V. Martin, A. Aquila, K. Nass, +A. Barty and H. N. Chapman, "CrystFEL: a software suite for snapshot serial crystallography" (2012), +J. Appl. Cryst. 45, 335-341 [doi:10.1107/S0021889812002312](https://doi.org/10.1107/S0021889812002312). + +**[GEMMI](https://github.com/project-gemmi/gemmi)** — symmetry operations, unit-cell and +structure-factor machinery, and MTZ / XDS_ASCII I/O. Vendored in `gemmi_gph/`, so it also carries a +licence obligation. M. Wojdyr, "GEMMI: A library for structural biology" (2022), J. Open Source +Softw. 7, 4200 [doi:10.21105/joss.04200](https://doi.org/10.21105/joss.04200). + +**Data-quality statistics** follow the established conventions rather than any one program: R_meas +and R_pim, CC1/2 and CC\*, and the reporting of I/sigma(I). K. Diederichs and P. A. Karplus, "Improved +R-factors for diffraction data analysis in macromolecular crystallography" (1997), Nat. Struct. Biol. +4, 269-275 [doi:10.1038/nsb0497-269](https://doi.org/10.1038/nsb0497-269); P. A. Karplus and +K. Diederichs, "Linking crystallographic model and data quality" (2012), Science 336, 1030-1033 +[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231); K. Diederichs and +P. A. Karplus, "Better models by discarding data?" (2013), Acta Cryst. D69, 1215-1222 +[doi:10.1107/S0907444913001121](https://doi.org/10.1107/S0907444913001121). + +**Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature: +D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography: +Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75 +[doi:10.1107/S0108767388009596](https://doi.org/10.1107/S0108767388009596); D. Schwarzenbach, +S. C. Abrahams, H. D. Flack, E. Prince and A. J. C. Wilson, "Statistical descriptors in +crystallography. II. Report of a Working Group on Expression of Uncertainty in Measurement" (1995), +Acta Cryst. A51, 565-569 [doi:10.1107/S0108767395002340](https://doi.org/10.1107/S0108767395002340). diff --git a/_sources/CBOR.md.txt b/_sources/CBOR.md.txt new file mode 100644 index 00000000..93f6ae24 --- /dev/null +++ b/_sources/CBOR.md.txt @@ -0,0 +1,368 @@ +# CBOR messages + +To communicate between FPGA-equipped receiver system and writers, +Jungfraujoch is using binary CBOR encoding with tinycbor library (Intel). +The protocol is based on and compatible with [DECTRIS Stream2](https://github.com/dectris/documentation/tree/main/stream_v2). +There are minor differences at the moment: + +* LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed +* Few fields are currently absent +* Extra fields are present beyond DECTRIS standard +* There are calibration and metadata messages defined beyond DECTRIS specification + +## Start message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "start" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| detector_distance | float | Detector distance \[m\] | | +| detector_translation | Array(float) | Detector translation vector \[m\] | X | +| beam_center_x | float | Beam center in X direction \[pixels\] | X | +| beam_center_y | float | Beam center in Y direction \[pixels\] | X | +| countrate_correction_enabled | bool | Countrate correction enabled | X | +| flatfield_enabled | bool | Flatfield enabled | X | +| number_of_images | uint64 | Number of images in the series | X | +| image_size_x | uint64 | Image width \[pixels\] | X | +| image_size_y | uint64 | Image height \[pixels\] | X | +| incident_energy | float | X-ray energy \[eV\] | X | +| incident_wavelength | float | X-ray wavelength \[Angstrom\] | X | +| incident_wavelength_spread | float (optional) | FWHM of the X-ray wavelength distribution \[Angstrom\] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic | | +| frame_time | float | Frame time, if multiple frames per trigger \[s\] | X | +| count_time | float | Exposure time \[s\] | X | +| saturation_value | int64 | Maximum valid sample value | X | +| error_value | int64 (optional) | Value used in images to describe pixels that are in error state or missing | | +| pixel_size_x | float | Pixel width \[m\] | X | +| pixel_size_y | float | Pixel height \[m\] | X | +| sensor_thickness | float | Sensor thickness \[m\] | X | +| sensor_material | string | Sensor material | X | +| arm_date | date | Approximate date of arming | X | +| pixel_mask_enabled | bool | Pixel mask applied on images | X | +| detector_description | string | Name of the detector | X | +| detector_serial_number | string | Detector serial number | X | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| fluorescence | object (optional) | X-ray fluorescence spectrum collected at start | | +| - energy | Array(float) | Energy of measuring point \[eV\] | | +| - data | Array(float) | Fluorescence scan result `data` \[arbitrary units\]; must be strictly the same length as energy | | +| goniometer | Map | Definition of rotation axis (optional) | X | +| - `AXIS` | string | Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch | X | +| - - increment | float | Rotation axis increment (per image) in degree \[deg\] | X | +| - - start | float | Rotation axis start angle \[deg\] | X | +| - - axis | Array(float) | Vector for the rotation axis | | +| - - helical_step | Array(float) | Translation for helical scan for 1 image \[m\] | | +| - - screening_wedge | Array(float) | Wedge for screening \[deg\] (increment would correspond to difference between screening points) | | +| grid_scan | object | Grid scan definition (optional and exclusive with rotation axis) | | +| - n_fast | uint64 | Number of elements along fast axis | | +| - n_slow | uint64 | Number of elements along slow axis | | +| - step_x_axis | float | Step along X axis, can be negative \[m\] | | +| - step_y_axis | float | Step along Y axis, can be negative \[m\] | | +| - snake_scan | bool | Snake scan (rows alternate direction) | | +| - vertical_scan | bool | Vertical scan (enabled: fast direction = Y, disabled: fast direction = X) | | +| jungfrau_conversion_enabled | bool (optional) | Applying JUNGFRAU pixel conversion (to photons or keV) | | +| jungfrau_conversion_factor | float (optional) | Factor used for JUNGFRAU conversion \[eV\] | | +| geometry_transformation_enabled | bool (optional) | Transformation from detector module geometry (512x1024) to full detector geometry | | +| pixel_mask | Map(string -> Image) | Pixel mask - multiple in case of storage cells | X | +| channels | Array(string) | List of image channels | X | +| max_spot_count | uint64 | Maximum number of spots identified in spot finding | | +| max_extra_lattices | uint64 | Maximum number of extra lattices | | +| storage_cell_number | uint64 (optional) | Number of storage cells used by JUNGFRAU | | +| storage_cell_delay | Rational | Delay of storage cells in JUNGFRAU | | +| threshold_energy | Map(string -> float) | Per-channel threshold energy \[eV\] (map of channel name to value) | | +| image_dtype | string | Pixel bit type (e.g. uint16) | X | +| unit_cell | object (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| az_int_q_bin_count | uint64 | Number of azimuthal integration bins in the radial direction | | +| az_int_phi_bin_count | uint64 | Number of azimuthal integration bins in the phi angle direction | | +| az_int_bin_to_q | Array(float) | Q value for each azimuthal integration bin \[angstrom^-1\] | | +| az_int_bin_to_two_theta | Array(float) | Two theta angle value for each azimuthal integration bin \[deg\] | | +| az_int_bin_to_phi | Array(float) | Phi value for each azimuthal integration bin \[deg\] | | +| az_int_map | Image | Mapping between pixel and bin number | | +| summation | uint64 | Factor of frame summation | | +| user_data | string | JSON serialized to string that can contain the following fields (all fields are optional): | X | +| - file_prefix | string | File prefix | | +| - images_per_file | uint64 | Number of images written per file | | +| - images_per_trigger | uint64 | Number of images collected per trigger | | +| - source_name | string | Facility name | | +| - source_type | string | Type of X-ray source (use NXsource/type values, for example "Synchrotron X-ray Source" or "Free-Electron Laser") | | +| - instrument_name | string | Instrument name | | +| - sample_name | string | Name of the sample | | +| - user | any valid JSON | Value of header_appendix provided at collection start to Jungfraujoch | | +| - attenuator_transmission | float | Attenuator transmission \[\] | | +| - total_flux | float | Total flux \[ph/s\] | | +| - space_group_number | uint64 | Space group number | | +| - summation_mode | string | Summation mode (internal\|fpga\|cpu) | | +| - overwrite | bool | Overwrite existing HDF5 files | | +| - file_format | int | File writer format: 0 = only data files, 1 = NXmx legacy soft links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF, 5 = TIFF, 6 = no file written | | +| - roi | Array(object) | ROI configurations; each element is one of: | | +| | | type "box": xmin, xmax, ymin, ymax (numbers) | | +| | | type "circle": r, x, y (numbers) | | +| | | type "azim": qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector | | +| - gain_file_names | Array(string) | Names of JUNGFRAU gain files used for the current detector | | +| - write_master_file | bool | With multiple sockets, it selects which socket will provide master file | | +| - write_images | bool | Write images in the HDF5 file (if false, will only write metadata) | | +| - data_reduction_factor_serialmx | uint64 | Data reduction factor for serial MX | | +| - experiment_group | string | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number | | +| - jfjoch_release | string | Jungfraujoch release number | | +| - socket_number | uint64 | Number of ZeroMQ socket (on `jfjoch_broker` side) used for transmission | | +| - bit_depth_readout | uint64 | Bit depth of the detector readout | | +| - writer_notification_zmq_addr | string | ZeroMQ address to inform `jfjoch_broker` about writers that finished operation | | +| - xfel_pulse_id | uint64 | Pulse IDs are recorded for images | | +| - ring_current_mA | float | Ring current at the start of the measurement | | +| - sample_temperature_K | float | Sample temperature \[K\] | | +| - detect_ice_rings | bool | Ice ring detection feature is enabled | | +| - indexing_algorithm | string | Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none | | +| - geom_refinement_algorithm | string | Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center | | +| - poni_rot1 | float | Tilt of the detector rot1 according to PyFAI PONI convention \[rad\] | | +| - poni_rot2 | float | Tilt of the detector rot2 according to PyFAI PONI convention \[rad\] | | +| - poni_rot3 | float | Tilt of the detector rot3 according to PyFAI PONI convention \[rad\] | | + +See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) for definition of Image as MultiDimArray with optional compression. + +## Image message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|-----------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "image" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| real_time | Rational | Exposure time | X | | +| start_time | Rational | Exposure start time (highly approximate) | X | | +| end_time | Rational | Exposure end time (highly approximate) | X | | +| spots | Array(object) | Spots: | | | +| - x | float | observed position in x (pixels) | | | +| - y | float | observed position in y (pixels) | | | +| - I | float | intensity (photons) | | | +| - maxc | int64 | max count (photons) | | | +| - ice_ring | bool | spot in resolution range for ice rings | | | +| - indexed | bool | indexed solution | | | +| - latt | int64 | Lattice to which the peak belongs (negative number = not indexed) | | | +| - image | int64 | image number the spot belongs to | | | +| - h | int64 | Miller index (indexed spots only) | | | +| - k | int64 | Miller index (indexed spots only) | | | +| - l | int64 | Miller index (indexed spots only) | | | +| - dist_ewald | float | distance to Ewald sphere \[Angstrom^-1\] (indexed spots only) | | | +| reflections | Array(object) | Reflections: | | | +| - h | int64 | Miller index | | | +| - k | int64 | Miller index | | | +| - l | int64 | Miller index | | | +| - x | float | prediced position in x (pixels) | | | +| - y | float | predicted position in y (pixels) | | | +| - obs_x | float | observed position in x (pixels) | | | +| - obs_y | float | observed position in y (pixels) | | | +| - d | float | resolution \[Angstrom\] | | | +| - I | float | integrated intensity (photons) | | | +| - bkg | float | mean background value (photons) | | | +| - var_bkg | float | non-signal (background) part of sigma^2, carried to the merge (photons^2) | | | +| - sigma | float | standard deviation, estimated from counting statistics (photons) | | | +| - image | float | image number (present for each spot) | | | +| - rp | float | Distance to Ewald sphere \[Angstrom^-1\] | | | +| - rlp | float | Reciprocal Lorentz and polarization corrections | | | +| - partiality | float | Partiality of the reflection | | | +| - phi | float | phi angle from XDS: difference from middle of current frame, not absolute \[deg\] | | | +| - zeta | float | Lorentz zeta factor (reciprocal-space geometry term) | | | +| - image_scale_corr | float | Per-image scale correction; I_true = image_scale_corr * I | | | +| spot_count | uint64 | Spot count | | | +| spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | +| spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | +| az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| | | NaN is used for empty bins and has to be taken care by the receiver | | | +| az_int_profile_std | Array(float) | Standard deviation for azimuthal integration. (NaN for less than 2 samples) | | | +| az_int_profile_count | Array(uint64) | Number of pixels contributing to azimuthal bin | | | +| indexing_result | bool | Indexing successful | | | +| indexing_lattice_count | int64 | Number of indexing lattices found for this image | | | +| indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| indexing_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | | +| indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| profile_radius | float | Profile radius of the image - describes distance of observed reflections from the Ewald sphere \[Angstrom^-1\] | | | +| integrated_reflections | int64 | Count of integrated reflections | | | +| mosaicity | float | Angular range of spots in image from a rotation scan \[degree\] | | | +| b_factor | float | Estimated B-factor (Angstrom^2) | | | +| compression_time | float | Time spent on compression/decompressing image \[s\] | | | +| preprocessing_time | float | Time spent on preparing the image for analysis \[s\] | | | +| azint_time | float | Time spent on azimuthal integration \[s\] | | | +| spot_finding_time | float | Time spent on spot finding \[s\] | | | +| indexing_time | float | Time spent on indexing \[s\] | | | +| refinement_time | float | Time spent on refinement of indexing solution and experimental geometry \[s\] | | | +| index_analysis_time | float | Time spent on analyzing idnexing solution, calculating profile radius and mosaicity \[s\] | | | +| bragg_prediction_time | float | Time spent on predicting Bragg spots \[s\] | | | +| integration_time | float | Time spent on Bragg integration \[s\] | | | +| image_scale_time | float | Time spent on on-the-fly scaling \[s\] | | | +| processing_time | float | Total processing time \[s\] | | | +| xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| lattice_type | object | Bravais lattice classification of the indexing result (present only if available) | | X | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | +| jf_info | uint64 | Detector info field | | | +| receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| receiver_free_send_buf | uint64 | Receiver internal number of available buffer locations | | | +| receiver_buf_in_sending | uint64 | Receiver internal number of buffer locations currently in sending/writing | | | +| receiver_buf_in_preparation | uint64 | Receiver internal number of buffer locations currently in processing | | | +| storage_cell | uint64 | Storage cell number | | | +| saturated_pixel_count | uint64 | Saturated pixel count | | | +| pixel_sum | uint64 | Sum of all pixels, excl. error and saturation | | | +| error_pixel_count | uint64 | Error pixel count | | | +| strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| min_viable_pixel_value | int64 | Minimal pixel value, excl. error and saturation | | | +| max_viable_pixel_value | int64 | Maximal pixel value, excl. error and saturation | | | +| resolution_estimate | float | Diffraction resolution estimation \[Angstrom\] | | X | +| data_collection_efficiency | float | Image collection efficiency \[\] | | | +| packets_expected | uint64 | Number of packets expected per image (in units of 2 kB) | | | +| packets_received | uint64 | Number of packets received per image (in units of 2 kB) | | | +| bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] | | | +| ice_ring_score | float | Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | | | +| spot_count_ice_control | float | Spots in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands' own q width (control for spot_count_ice_rings) | | | +| beam_corr_x | float | Beam center correction X applied during processing \[pixel\] | | X | +| beam_corr_y | float | Beam center correction Y applied during processing \[pixel\] | | X | +| image_scale_factor | float | Scaling result: Image scale factor (g) | | X | +| image_scale_mosaicity | float | Scaling result: Image scale mosaicity \[deg\] | | X | +| image_scale_cc | float | Scaling result: Image scale CC | | X | +| adu_histogram | Array(uint64) | ADU histogram | | | +| roi_integrals | object | Results of ROI calculation | | X | +| - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - pixels | uint64 | Valid pixels in ROI area | | | +| - max_count | int64 | Highest count in ROI area \[photons\] | | | +| - x_weighted_sum | int64 | ROI pixel X position multiplied by photon count \[photons * pixels\] | | | +| - y_weighted_sum | int64 | ROI pixel Y position multiplied by photon count \[photons * pixels\] | | | +| user_data | string | Optional user defined text information - this is image_appendix serialized to JSON format | X | | +| data | Map(string -> Image) | Image | X | | + +## Metadata message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|------------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "metadata" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| images | Array(object) | Array of images (order and size of the array are not guaranteed) | X | | +| - image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| - original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| - real_time | Rational | Exposure time | X | | +| - start_time | Rational | Exposure start time (highly approximate) | X | | +| - end_time | Rational | Exposure end time (highly approximate) | X | | +| - spot_count | uint64 | Spot count | | | +| - spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| - az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| - indexing_result | bool | Indexing successful | | | +| - indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| - indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| - xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| - xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| - jf_info | uint64 | Detector info field | | | +| - receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| - receiver_free_send_buf | uint64 | Receiver internal number of available send buffers | | | +| - storage_cell | uint64 | Storage cell number | | | +| - saturated_pixel_count | uint64 | Saturated pixel count | | | +| - error_pixel_count | uint64 | Error pixel count | | | +| - strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| - data_collection_efficiency | float | Image collection efficiency \[\] | | | +| - bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] (with solid angle/polarization corrections, if applied) | | X | +| - resolution_estimate | float | Diffraction resolution estimation | | X | +| - adu_histogram | Array(uint64) | ADU histogram | | X | +| - roi_integrals | object | Results of ROI calculation | | X | +| - - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - - pixels | uint64 | Valid pixels in ROI area | | | +| - - max_count | int64 | Highest count in ROI area \[photons\] | | | + +## End message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "end" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| end_date | string | Approximate end date | | +| max_image_number | uint64 | Number of image with the highest number; counted from 1 to distinguish zero images and one image | | +| images_collected | uint64 | Number of images collected | | +| images_sent_to_write | uint64 | Number of images sent to writer; if writer queues were full, it is possible this is less than images collected | | +| data_collection_efficiency | float | Overall network packets collected / network packets expected | | +| az_int_result | Map(text->Array(float)) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | +| adu_histogram | Map(text->Array(uint64)) | ADU values histogram | | +| adu_histogram_bin_width | uint64 | Width of bins in the above histogram \[ADU\] | | +| max_receiver_delay | uint64 | Internal performance of Jungfraujoch | | +| bkg_estimate | float | Mean background estimate for the whole run | | +| indexing_rate | float | Mean indexing rate for the whole run | | +| unit_cell | object (optional) | Unit cell of the system, based on the actual experiment: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| rotation_lattice_type | object | Bravais lattice classification of the total rotation solution over the run, if available; same schema as `lattice_type` | | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | +| rotation_lattice | Array(9 * float) | Real-space lattice basis, flattened 3x3 in row-major order | | +| rotation_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | +| data_collection_efficiency_image | Array(float) | Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context | | +| spot_count | Array(int32) | Per-image spot count | | +| spot_count_ice_ring | Array(int32) | Per-image number of spots within identified ice-ring resolution ranges | | +| spot_count_low_res | Array(int32) | Per-image number of low-resolution spots | | +| spot_count_indexed | Array(int32) | Per-image number of spots fitting indexing solution | | +| image_indexed | Array(uint8) | Per-image indexing result; 0 = not indexed, nonzero = indexed | | +| v_bkg_estimate | Array(float) | Per-image background estimate | | +| ice_ring_score | Array(float) | Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice) | | +| spot_count_ice_control | Array(float) | Per-image spot count in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands' q width | | +| ice_ring_score_mean | float | Mean ice-ring score for the whole run (1 = no ice) | | +| profile_radius | Array(float) | Per-image profile radius \[Angstrom^-1\] | | +| mosaicity | Array(float) | Per-image mosaicity \[degree\] | | +| bFactor | Array(float) | Per-image estimated B-factor \[Angstrom^2\] | | +| resolution_estimate | Array(float) | Per-image diffraction resolution estimate \[Angstrom\] | | +| min_viable_pixel_value | Array(int64) | Per-image minimum valid pixel value, excluding error/saturated pixels | | +| max_viable_pixel_value | Array(int64) | Per-image maximum valid pixel value, excluding error/saturated pixels | | +| saturated_pixel_count | Array(int32) | Per-image saturated pixel count | | +| error_pixel_count | Array(int32) | Per-image error pixel count | | +| image_scale_factor | Array(float) | Per-image scale factor, if scaling/merging was performed | | +| integrated_reflections | Array(int32) | Per-image count of integrated reflections | | +| indexed_lattice_count | Array(int32) | Per-image count of indexed lattices | | +| niggli_class | Array(uint8) | Per-image Niggli class identifier for indexed images; 0 if unavailable | | +| pixel_sum | Array(int64) | Per-image sum of all valid pixels, excluding error/saturated pixels | | +| image_scale_mosaicity | Array(float) | Scaling result: Image scale mosaicity \[deg\] | | +| image_scale_cc | Array(float) | Scaling result: Image scale CC | | + +End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted. + +## Calibration message + +| Field name | Type | Description | Present in DECTRIS format | +|--------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "calibration" | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| data | Map(string -> Image) | Calibration map (only single pedestal array per message) | | + +## User data +In many cases there is an interest from facilities to forward more metadata, than available explicitly in the Jungfraujoch. +For this reason two fields can be provided: `header_appendix` (sent with start message) and `image_appendix` (send with image message). +To increase flexibility, both appendices can contain any valid JSON message. +These appendices are serialized into string and stored in CBOR messages as `user_data`. + +Notably for start message, `user_data` can contain more information (non-DECTRIS compliant metadata). +Therefore `user_data` is serialized by Jungfraujoch as CBOR object. There is member `user` which contains `header_appendix` defined in OpenAPI of Jungfraujoch. + +### Notes on images and compression + +- Images are encoded as DECTRIS MultiDimArray with typed array tags: + - For RGB: shape \[3, height, width\], type: u8 + - For grayscale: shape \[height, width\], type according to bit depth and sign (e.g., uint16 LE) +- Compression: + - Uncompressed: raw CBOR byte string + - Bitshuffle+LZ4: tag with \["bslz4", elem_size, bytes\] + - Bitshuffle+Zstandard: tag with \["bszstd", elem_size, bytes\] + +### Notes on typed arrays + +Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays. + +Common tags used in this protocol include: + +- float32 little-endian arrays for `Array(float)` +- uint8 arrays for compact boolean/integer flags such as `image_indexed` +- int32 little-endian arrays for per-image counts +- int64 little-endian arrays for large per-image integer values +- uint64 little-endian arrays for histograms \ No newline at end of file diff --git a/_sources/CHANGELOG.md.txt b/_sources/CHANGELOG.md.txt new file mode 100644 index 00000000..25716849 --- /dev/null +++ b/_sources/CHANGELOG.md.txt @@ -0,0 +1,1155 @@ +# Changelog +## 1.0.0 +### 1.0.0-rc.161 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* **rugnux: significantly better quality of results, and faster.** A large rework of integration, scaling, merging, geometry refinement and space-group determination, together with measurements the program previously made no attempt at - the direct beam before indexing, the beam stop, the goniometer rotation scale, and the stretches of a sweep the crystal did not deliver. A rotation dataset typically gains observations at better and R_meas, and every `mx` and `scale` run writes a `_report.txt` results report modelled on XDS's `CORRECT.LP`. Many defaults moved with it: spot detection is self-calibrating, beam-stop detection and rotation geometry post-refinement are on, resolution limits default to as far as the detector reaches, and ice-ring handling engages only where the crystal is measured to have ice. +* **jfjoch_viewer:** the beam-stop shadow, the detector calibration and the beam-centre measurement are reachable from "Analyze dataset"; the settings panel reports how the sample moved and how polarized the beam was; image rendering and interaction are faster. +* **Performance:** bitshuffle+LZ4 images are decoded on the GPU rather than on the host, with the bitshuffle inverse fused into preprocessing so the decompressed frame is never held in device memory. +* **Broker, writer, packaging and build:** image-slot lifetime and locking fixes, per-image datasets sized by the images actually written, the Debian/Ubuntu broker package renamed to `jfjoch`, and `image_analysis` compiling under MSVC again. + +**Breaking change to the rugnux command line:** +* `--azint-only` and `--scale` are **removed**, replaced by `--mode azint` and `--mode scale`; the full pipeline is `--mode mx` and remains the default. A script passing the old flags now fails with the list of valid modes rather than silently running the wrong one. +* `-t`/`--stride` is **refused on rotation data**: skipping frames cuts every reflection's rocking curve, so the combined fulls and their partiality would be measured over frames the sweep never recorded. Select a contiguous range with `-s`/`-e` instead. `--mode azint` and `--force-still` still take a stride. + +**Breaking changes to OpenAPI** - regenerate the client (`jfjoch-client` 1.0.0-rc.161, `frontend/src/client`) or read the affected fields as optional: +* `image_scale_b` is removed from the `plot_type` enum, so a client requesting that plot now gets an error rather than a curve. +* `azim_int_settings.high_q_recipA`, `spot_finding_settings.high_resolution_limit` and `spot_finding_settings.low_resolution_limit` are no longer `required`. All three mean "no limit at that end" when unset and are omitted from the response instead of carrying a placeholder value, which raises in a client generated from an rc.160-or-earlier spec. A value of 0 is still accepted and means the same thing. + +**Breaking changes to the stored formats** - a consumer reading these fields must treat them as optional: +* The per-image image-scale B factor is no longer computed, so `/entry/MX/imageScaleBFactor` is absent from newly written HDF5 files and the corresponding key is absent from the CBOR DataMessage and END blocks. Files written by rc.160 and earlier still contain it and still open; nothing in the pipeline reads it any more. +* `_reflns.jfjoch_diffrn_ISa` now carries the whole-range `1/sqrt(a*b)` that XDS's ISa denotes, and the error-model `a` and `b` are reported in XDS's convention; the strong-reflection asymptote moves to `_reflns.jfjoch_diffrn_ISa_asymptotic`. **A file written by an earlier version carries the asymptote under the plain `ISa` name.** + +### 1.0.0-rc.160 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* rugnux: Rotation **geometry post-refinement** is now on by default (`--rotation-no-postrefine` to disable; also a viewer checkbox). A first pass integrates at the header geometry, then the shared detector distance + beam centre and the crystal cell/goniometer-axis are post-refined over all frames (cross-validated, committed only for a small < 1 % move, with the gauge-weak beam centre restrained toward the header); a second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical `_*` output; the header-geometry pass is kept as `_01_*`. +* rugnux: Optional per-batch **relative-B** correction for rotation (`--relative-b[=deg]`, default 10°-of-rotation batches when bare, off otherwise) - a cross-validated, curvature-smoothed resolution×dose correction beyond the single global decay slope. +* rugnux: Always-on **radiation-damage report** for rotation - the per-image scale correlation-to-merge and mosaicity versus dose, plus the relative B-factor change over the run (first→last) as a scalar and a per-batch relative-B curve, printed to the log and written to the merged mmCIF. Report-only; it never alters the merge. +* rugnux: De-novo space-group search ranks candidate lattice **centerings by net absences** (systematically-absent minus violating), not the gross absent count, fixing an over-centering of a genuinely C-centred lattice to F. +* rugnux: Record the **producing software** (name and version) and the refined **detector distance and beam centre** in the merged mmCIF (and the software in the MTZ history). +* Bragg integration: Carry the box-sum observed centroid through the profile-fit path, so the observed spot centroid is emitted in every integrator mode. +* rugnux: Report **ISa** as the counting-subtracted strong-reflection asymptote, not `1/b` of the whole-range fit; it also sets the merged-sigma floor. CC1/2, R-meas and per-obs sigmas unchanged. +* Frontend: Azimuthal-integration Q fields (Q spacing / Low Q / High Q) accept 5 decimals (was 3), matching the 1e-5 `q_spacing` minimum; number-field precision is now configurable. +* rugnux: Add a dataset-wide **Wilson B-factor** estimate to the merged output (mmCIF, stats table, log); the per-image viewer Wilson B emits NaN for implausible fits. +* rugnux: De-novo space-group search vetoes a merohedral-twin over-promotion whose systematic error-model `b` balloons past a calibrated bound (keeps R3 as R3, not R32). +* rugnux: De-novo space-group search decides lattice **centering** from the strength (mean I/sigma) of the systematically-absent class, not a per-reflection violation count. +* rugnux: Recover lattice **centering** on weak / low-energy data via a floor-independent test (rate of significant absent vs present reflections), fixing a missed I-centring at 5/13 keV. +* rugnux: Report anomalous signal-to-noise **SigAno** = <|I(+)-I(-)|>/ per shell and overall (mmCIF PDBx items + stats-table column); anomalous merges only. +* rugnux: De-novo space-group search recovers a genuine high-symmetry group on weak data with a broken sigma model by confirming on the systematic-`b` test alone (restores an F432 case). +* rugnux: Print the adopted **space group and unit cell** as a one-line summary at the end of the run (de-novo or user-fixed `-S`). +* rugnux: Score the radiation-damage **decay** cross-validation on a sigma-independent (R-meas-like) metric, so a spurious slope can't pass by reshaping sigmas. +* rugnux: Fix de-novo rotation indexing committing a spurious axis-multiple supercell (collapsing to P1) via a cross-scheme smaller-cell tie-break on near-integer volume ratios. +* rugnux: Widen refined-cell angle bounds to [30, 150] deg (rotation candidate and per-frame stills refinement); check refined angles against the reference cell. +* rugnux: `-S`/`--space-group` now accepts a Hermann-Mauguin symbol (e.g. `P43212`) as well as a space-group number. +* rugnux: Warn when the chosen cell/space group carries an indexing (merohedral) ambiguity needing a reference to resolve. +* Indexing: Requesting the FFTW (CPU) indexer on a GPU node now fails with a clear, actionable message (rotation always uses the GPU FFT indexer there). +* rugnux: Stills `--refine-geometry[=N|off]` - first-pass bundle-adjust of beam/distance/cell then re-index (default ON with a reference cell); accepts reference `F`/`FP` columns. +* rugnux: Per-image geometry refinement `-r flex` tries all three algorithms per image and keeps the best (old name `multi` kept as an alias). +* rugnux: Experimental stills partiality `--still-partiality` (Gaussian excitation-error) and `--partiality-uncertainty ` down-weighting the least-complete partials. +* rugnux: Default the stills Bragg-integration box to r=6 (integration radii 6, 8, 12). +* rugnux: Self-referenced stills scale in a single pass (fixing a weak-data collapse); a reference MTZ (`-z`) fixes SG/cell/ambiguity but never anchors the scale (stills and rotation). +* rugnux: Cap normalised intensity (E^2) on second-lattice overlaps in the de-novo space-group search, so strong overlaps don't skew the symmetry decision. +* rugnux: Fix `--scale` on a self-contained `_process.h5` (stored reflections and error model reload correctly). +* rugnux: Add `--spot-low-resolution ` (default 50 A) and `--min-pix-per-spot ` (default 2) to tune spot finding on weak serial data. +* jfjoch_viewer: Expose stills processing settings in the settings dock, rename geometry-refinement `multi` to `flex`, and refit the initial image on resize. +* jfjoch_writer: Remove the CBF and TIFF image writers - only NXmx HDF5 is written (all three layouts remain). +* Reader: Treat a negative `total_flux` in a stored dataset as unknown/absent rather than a valid flux. +* Packaging: Build the self-contained Linux viewer against a static libdbus with glib disabled; add parallel image-build and in-container viewer-verification scripts. +* rugnux: Write anomalous data as a standard CCP4 anomalous MTZ (one row per reflection: `IMEAN`, `I(+)`/`I(-)`, `F`/`F(+)`/`F(-)`), readable by aimless/mtz2sca/ANODE. +* rugnux: Always write merged reflections as both `.mtz` and `.cif`; the `--scaling-output` selector and text `.hkl` output are removed. +* rugnux: Add a detector-plane **modulation** (flat-field) correction surface to rotation scaling (cross-validated, on by default; `--no-scaling-corrections` disables all), dropping R-meas. +* rugnux: Add optional **stills detector-plane modulation** (`--stills-modulation`, default off) - the same cross-validated surface for the on-the-fly stills path. +* Bragg integration: Local background is now a **symmetric trimmed mean** of the ring (`--background-trim `, default 0.10; monochromatic), improving and resolution-edge CC1/2. + +### 1.0.0-rc.159 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* rugnux: Add `--model model.pdb` - score the merged data against an atomic model and compute initial maps. It reports R-work/R-free (scaling the model to the observed amplitudes with an overall scale, an anisotropic B and a flat bulk solvent - the standard few-parameter model, so a batch of maps stays directly comparable) and writes 2Fo-Fc / Fo-Fc electron-density maps (CCP4) plus a map-coefficient MTZ. The structure itself is not refined; the model is only re-fractionalised into the data cell. +* rugnux: The merged reflection output now carries French-Wilson amplitudes (|F| and its sigma) next to the intensities - MTZ `F`/`SIGF`, mmCIF `_refln.F_meas_au`, and the text HKL - computed with the correct centric/acentric Wilson prior and epsilon multiplicity, so a downstream program (e.g. phenix.refine) can refine against amplitudes. The intensity columns are unchanged. +* rugnux: R-free test-set flags are now assigned deterministically and consistently across symmetry - a Bijvoet pair I(+)/I(-) is never split between the work and free sets, and the assignment is a reproducible per-hkl hash that depends only on the reflection index, so every dataset of one crystal form gets the same ~5% free set (what a multi-dataset campaign such as PanDDA needs). On small data the fraction is floored so the test set stays large enough for a stable R-free (~500 reflections, capped at 10%); it stays flat at 5% on ordinary data. When a reference MTZ carries a `FreeR_flag` column its test set is imported instead, letting a whole campaign inherit one shared free set. +* rugnux: A reference MTZ (`--reference-mtz`) can now fix the space group and cell for rotation data too (previously rejected), without being used to scale - the rotation merge stays self-consistent. When the crystal has an indexing (merohedral) ambiguity - a lattice symmetry higher than its Laue symmetry, e.g. P3/P4/P6/C2 - the reference also resolves it: each candidate reindexing (identity plus the twin-law cosets of the metric symmetry) is scored by its intensity correlation against the reference and the data are re-merged in the best-correlating one. This is a metric-preserving relabelling of hkl (the cell is unchanged) and a no-op for a holohedral crystal (which has no twin laws). +* rugnux: `--model` validation now aligns the data to the model before scoring - the observed reflections are reindexed into the model's enantiomorph when the two differ only by hand (indistinguishable from merged intensities). A merohedral indexing ambiguity is resolved against the reference MTZ when one is given (so a whole campaign shares one indexing convention); only with a model and no reference does validation fall back to fitting each candidate reindexing and keeping the lowest R-free. +* rugnux: De-novo symmetry - recover a genuine high-symmetry group whose data are imperfectly scaled. Such a merge's within-orbit chi² lands just past the self-consistency bound (each real symmetry step adds a little systematic scatter), right where a merohedral twin also lands, so the chi² ratio alone cannot separate them. The candidate is now rescued when the extra intensity-proportional systematic error it invokes stays small relative to the confirmed subgroup - a genuine symmetry step gains multiplicity without inflating the merge error model's b, whereas a twin forces non-equivalent reflections together and b balloons. Fixes a cubic case (I23 instead of I222) with no change to any other crystal in the test battery, including the twins that must stay in their lower symmetry. +* Docs: Document the French-Wilson amplitude estimation, R-free flagging, reference-based space-group/ambiguity resolution, and model-based validation/maps in CPU_DATA_ANALYSIS.md. +* Frontend: The status-bar pill now shows a progress bar during detector calibration (previously only during measurement), and the calibration state and its button are labelled "Calibration"/"CALIBRATE" (the internal `Pedestal` state name is unchanged for back-compatibility). + +### 1.0.0-rc.158 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (`cos^3` of that angle) instead of `cos^3(2*theta)`, so it is correct for a tilted detector and matches PyFAI `solidAngleArray` and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (`XtalOptimizer`) no longer silently ignores an imported PONI `rot3` (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured `rot3` through the full PONI rotation. +* jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting `\\`); write processing outputs next to the input file, with a Browse button and independent `_process.h5` / merged `.mtz`/`.cif` toggles; and show the determined space group in the merge-statistics window. +* jfjoch_viewer: Connect to a broker over `https` (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow. +* jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source. +* rugnux: Accept an absolute `-o` output prefix in offline processing. +* rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged). +* rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing. +* rugnux: De-novo symmetry robustness - don't over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don't flag twinning in holohedral Laue classes where no twin law can exist. +* jfjoch_writer: Fold the refined beam centre into the NXmx detector `translation` vector too (not only the informational `beam_center` fields), so a reprocessed `_process.h5` has a self-consistent refined geometry. +* Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames. +* Packaging: The self-contained Linux viewer `.tgz` now bundles cuFFT, so it runs without a system CUDA toolkit (`.deb`/`.rpm` are unchanged, distro-managed). +* Docs: Documentation updated to match the current analysis code and CLI. + +### 1.0.0-rc.157 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* rugnux: Rebrand the offline data-processing subsystem as `rugnux` and consolidate all offline analysis into the single `rugnux` binary - `jfjoch_process` is now `rugnux`, the former `jfjoch_azint` is now `rugnux --azint-only`, and `jfjoch_scale` is now `rugnux --scale` (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (`--no-merge` disables them), replacing the previous opt-in `-M, --scale-merge`. +* rugnux: CLI fixes - default `-N` to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require `--wavelength > 0`, and correct the reproduced command line and `--scale` reference-cell handling. +* rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics. +* rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection's rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below = 1. +* rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below `--min-captured-fraction` (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned). +* rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved `jfjoch` prefix. +* jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation. +* jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a "reduce frame rate" warning. +* jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint. +* jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed). +* jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types. +* jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails. +* jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak. +* Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation. +* CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*. + +### 1.0.0-rc.156 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output. +* jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed. +* jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections. +* jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw). +* jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path. +* jfjoch_process: Unify still and rotation processing on a single --force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental --reciprocal-profile option is removed. +* jfjoch_process: Add EXPERIMENTAL ice-ring detection (--detect-ice-rings) that excludes ice reflections from scaling. +* jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend. +* jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata. +* jfjoch_writer: Don't write empty grid-scan position arrays when the dataset has no images. +* Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don't fit rather than aborting), and add the jfjoch_recompress tool. +* jfjoch_viewer: Report "Multiple lattices detected" and grey out "Analyze dataset" on a live connection. +* jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons. +* CI: Build Windows (CUDA and non-CUDA) installers. +* CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz. + +### 1.0.0-rc.155 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D) +* jfjoch_viewer: Some graphical improvements. +* jfjoch_viewer: Simplify und unify data analysis settings. +* jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker "dies" in the middle of data acquisition. + +### 1.0.0-rc.154 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability) + +### 1.0.0-rc.153 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing +* jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files +* jfjoch_broker: Add ROI calculation in non-FPGA workflow +* jfjoch_broker: Fixes to TCP image pusher +* jfjoch_broker: Remove NUMA bindings +* jfjoch_broker: Improvements to indexing +* jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values +* jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors +* jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs +* jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results + +### 1.0.0-rc.152 +* jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5) +* jfjoch_viewer: Adjust Q bounds for azimuthal integration +* jfjoch_azint: Add tool to do quick azimuthal integration + +### 1.0.0-rc.151 +* jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname + +### 1.0.0-rc.150 +* jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin. +* jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins. + +### 1.0.0-rc.149 +* XDS plugin: Fix HDF5 mutex to run on multiple processors + +### 1.0.0-rc.148 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Improve azimuthal integration (add calculation) +* jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated) +* jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin + +### 1.0.0-rc.147 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015. +* jfjoch_viewer: Add reciprocal space viewer +* jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset +* jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress) + +### 1.0.0-rc.146 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension) +* jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset. +* jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling. +* jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file. +* jfjoch_scale: Dedicated tool for rescaling/merging existing data. +* jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel. + +WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being. + +### 1.0.0-rc.145 +This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144. + +* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future. +* XDS plugin: Improve performance of VDS reading. +* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD. +* jfjoch_writer: Clean-up of HDF5 routines to better handle issues. + +### 1.0.0-rc.144 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M) + +### 1.0.0-rc.143 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment + +### 1.0.0-rc.142 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8 +* Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent + +### 1.0.0-rc.141 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times +* frontend: Fix selection of FFTW in indexing settings + +### 1.0.0-rc.140 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection +* jfjoch_broker: Minor bug fixes for rare conditions +* jfjoch_process: Significantly improve performance + +### 1.0.0-rc.139 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on `/start` +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on '/start' +* jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite worklow preparation time +* jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received +* jfjoch_broker: Use nanoseconds internally for frame/image/readout time +* jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished) +* jfojch_broker: Improve profiling of different data analysis steps +* jfjoch_broker: Record integration reflection count +* jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds) +* jfjoch_broker: Fix bug where '/wait_till_done' got deadlocked +* jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets + +**Breaking changes**: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time. + +### 1.0.0-rc.138 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time +* jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations +* jfjoch_broker: Goniometer axis name is converted to lowercase +* jfjoch_broker: Fix bug, where wrong HTTP error codes were returned +* jfjoch_process: Improve sigma estimation during merging (K. Takaba) +* jfjoch_process: Modify spot finding thresholds +### 1.0.0-rc.137 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Better track time for each operation in the processing stack +* jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress) +* jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress) +* jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0) + +### 1.0.0-rc.136 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress). + +### 1.0.0-rc.135 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4) +* jfjoch_viewer: Improve image render performance + +### 1.0.0-rc.134 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long +* jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata +* XDS plugin: Add generation of Jungfraujoch plugin for XDS +* CI: Add tests with XDS and DIALS (`xia2.ssx`) + +### 1.0.0-rc.133 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Use httplib for HTTP server instead of Pistache +* jfjoch_broker: Drop OpenSSL support +* jfjoch_broker: Base work for multi-lattice support in the future +* jfjoch_broker: Improve recording time of data analysis steps +* jfjoch_writer: Save per-image information about data analysis timing +* Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib) + +### 1.0.0-rc.132 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* Documentation: Fix equation rendering + +### 1.0.0-rc.131 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS) +* jfjoch_viewer: Fix calibration (pedestal) images being open flipped +* jfjoch_process: Add space group detection (EXPERIMENTAL) + +### 1.0.0-rc.130 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Rotation indexer has two retries if failes +* jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot) +* jfjoch_broker: Integration calculates background mask based on R2 radius +* jfjoch_process: HDF5 files are not saved by default + +### 1.0.0-rc.129 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files) +* jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction +* jfjoch_broker: Image pusher statistics are accessible via the REST interface +* jfjoch_writer: Supports TCP image socket and for these auto-forking option + +### 1.0.0-rc.128 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Handle properly reuse of image buffer locations +* jfjoch_broker: Fix bug in counting idle slots +* jfjoch_broker: Force obtuse angle for monoclinic cells +* jfjoch_process: Change scaling refinement tolerance + +### 1.0.0-rc.127 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Default EIGER readout time is 20 microseconds +* jfjoch_broker: Multiple improvements regarding performance +* jfjoch_broker: Image buffer allows to track frames in preparation and sending +* jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer +* jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets +* jfjoch_writer: Fixes regarding properly closing files in long data collections +* jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code + +### 1.0.0-rc.126 +This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg. + +### 1.0.0-rc.125 +This is an UNSTABLE release. This version adds scalign and merging. These are experimental at the moment, and should not be used for production analysis. +If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Improve logic on switching on/off spot finding +* jfjoch_broker: Increase maximum spot count for FFBIDX to 65536 +* jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD) +* jfjoch_process: Add scalign and merging functionality - program is experimental at the moment and should not be used for production analysis +* jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection +* jfjoch_writer: Save more information about each reflection + +### 1.0.0-rc.124 +This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, +especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123. + +* jfjoch_broker: Improve refection position prediction and Bragg integration code. +* jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation. +* jfjoch_writer: Fix saving mosaicity properly in HDF5 file. +* jfjoch_viewer: Introduce high-dynamic range mode for images +* jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%) +* jfjoch_viewer: Zoom-in numbers have better readability + +### 1.0.0-rc.123 +This is an UNSTABLE release. + +* jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility +* jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M) +* jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M) +* jfjoch_viewer: Add more color schemes for better image readability +* HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable +* HDF5: suppress warning if path (upstream group) doesn't exists when checking if leaf exists + +### 1.0.0-rc.122 +This is an UNSTABLE release. + +* jfjoch_broker: Add thresholding to prefer shorter vectors after FFT +* jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress) +* jfjoch_broker: Update nlohmann::json to 3.12.0 +* jfjoch_viewer: Display file opening errors +* jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available + +### 1.0.0-rc.121 +This is an UNSTABLE release. + +* jfjoch_broker: Report changes in the image buffer, so viewer doesn't reload constantly +* jfjoch_viewer: Improve performance of loading images +* jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes +* jfjoch_viewer: Auto-foreground calculated with histogram +* jfjoch_viewer: Fix rare segmentation fault + +### 1.0.0-rc.120 +This is an UNSTABLE release. + +* jfjoch_broker: Improve performance of binary plot export + +### 1.0.0-rc.119 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Add binary export of data analysis plots over OpenAPI +* jfjoch_broker: Minor fixes to HTTP error handling +* jfjoch_viewer: Prefer binary plots over JSON plots +* jfjoch_viewer: Change foreground with F button + wheel +* jfjoch_viewer: Change way how angles are displayed +* jfjoch_viewer: Display resolution of the mouse cursor in top left corner + +### 1.0.0-rc.118 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement. +* jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations + +### 1.0.0-rc.117 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add ROI results to the dataset info plots +* jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment + +### 1.0.0-rc.116 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add binning options in the context menu + +### 1.0.0-rc.115 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Default spot finding settings can be configured via config JSON +* jfjoch_viewer: FFT analysis of data in the dataset plot + +### 1.0.0-rc.114 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Fix generating JPEG images with resolution estimation + +### 1.0.0-rc.113 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Improve handling of rotation indexing +* jfjoch_broker: More information saved in CBOR end message (WIP) +* jfjoch_writer: Save rotation indexing lattice parameters and Niggli class +* jfjoch_viewer: Remove (for now) primitive cell information +* jfjoch_viewer: Use angle for dataset info plot for rotation scans + +### 1.0.0-rc.112 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Experimental rotation (3D) indexing +* jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values + +### 1.0.0-rc.111 +This is an UNSTABLE release. + +* jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment) +* jfjoch_viewer: Fix auto contrast not refreshing image + +### 1.0.0-rc.110 +This is an UNSTABLE release. + +* jfjoch_broker: Add auto-contrast option for preview images +* Frontend: Add logo image +* jfjoch_viewer: Add logo image +* jfjoch_viewer: For image chart allow to set min value to zero +* jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure +* jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there) +* Documentation: Add logo image + +### 1.0.0-rc.109 +This is an UNSTABLE release. + +* jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard +* jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors + +### 1.0.0-rc.108 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN) +* jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default +* jfjoch_viewer: Fit to window of image is now default when size of the grid changes + +### 1.0.0-rc.107 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor polishing of new functionality +* jfjoch_broker: User NaN for empty azimuthal bins + +### 1.0.0-rc.106 +This is an UNSTABLE release. + +* jfjoch_viewer: Allow for multiple dataset info plots +* jfjoch_viewer: Highlight current element in grid + +### 1.0.0-rc.105 +This is an UNSTABLE release. + +* jfjoch_viewer: Clean-up widgets slightly +* jfjoch_viewer: Limit right panel to 600 pixels +* jfjoch_viewer: Parse crystal symmetry type +* jfjoch_viewer: Grid scan view takes color map and can be fit to zoom + +### 1.0.0-rc.104 +This is an UNSTABLE release. + +* jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier) +* jfjoch_viewer: Display grid scan results in 2D (work in progress) +* jfjoch_viewer: Improve auto-scaling on start of images (work in progress) +* jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots + +### 1.0.0-rc.103 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer +* jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates + +### 1.0.0-rc.102 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer + +### 1.0.0-rc.101 +This is an UNSTABLE release. + +* jfjoch_viewer: Auto load is better handling change of states +* jfjoch_viewer: Fix DBus registration +* jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update +* jfjoch_viewer: Calculate ROI in a more efficient way + +### 1.0.0-rc.100 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix dbus registration +* jfjoch_viewer: Remove background slider for diffraction image +* jfjoch_viewer: Adjustments for 2D azimuthal image viewer + +### 1.0.0-rc.99 +This is an UNSTABLE release. + +* jfjoch_broker: Fix output during mask data collection + +### 1.0.0-rc.98 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization + +### 1.0.0-rc.97 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask +* jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems) +* jfjoch_viewer: Add option to handle user pixel mask (experimental) +* jfjoch_viewer: More options for ROI +* jfjoch_viewer: Add window to display calibration + +### 1.0.0-rc.96 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* jfjoch_broker: Remove PNG preview, no dependency on libpng +* jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds) +* jfjoch_viewer: Show data collection time in dataset tooltip +* jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate) +* jfjoch_viewer: Auto foreground value +* Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds +* Remove OpenBLAS dependency + +### 1.0.0-rc.95 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* Add git-lfs to Rocky8 docker image + +Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved. + +### 1.0.0-rc.94 +This is an UNSTABLE release. + +* FFTIndexer: Add limit on angles to avoid colinear vectors +* Docker images: Add 3D Qt +* Gitea: Fixes to the pipeline + +### 1.0.0-rc.93 +This is an UNSTABLE release. + +* CI: Fixes to Gitlab based pipeline +* PCIe driver: Fix PCIe revision being hex number + +### 1.0.0-rc.92 +This is an UNSTABLE release. + +* jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample. + +### 1.0.0-rc.91 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated +* jfjoch_broker: Spot finding and integration predictions are ported to a GPU +* jfjoch_broker: Estimate resolution +* jfjoch_broker: Lattice search +* jfjoch_broker: Many more improvements in image analysis + +### 1.0.0-rc.90 +This is an UNSTABLE release. + +* jfjoch_broker: for indexing min index spots for a viable cells can be changed via OpenAPI +* jfjoch_viewer: Optional auto-reanalyze images +* jfjoch_writer: Add option where no files at all are saved +* Documentation: improvements + +### 1.0.0-rc.89 +This is an UNSTABLE release. + +* jfjoch_broker: Fix resolution estimation code +* jfjoch_broker: Fix Wilson B-factor calculation code +* jfjoch_viewer: Improve display of plots +* jfjoch_viewer: Fix segmentation fault +* jfjoch_viewer: Display missing metadata when using HTTP +* jfjoch_viewer: Fix bug when opening the same file twice + +### 1.0.0-rc.88 +This is an UNSTABLE release. + +* jfjoch_viewer: Add resolution estimation to the image information +* jfjoch_broker: Minor changes to resolution estimate routine + +### 1.0.0-rc.87 +This is an UNSTABLE release. + +* jfjoch_viewer: Display more image metadata (angle / exposure time) +* jfjoch_viewer: Improve I/sigma and B-factor plots +* jfjoch_broker: Estimate resolution based on visible spots + +### 1.0.0-rc.86 +This is an UNSTABLE release. + +* jfjoch_broker: Update logic when initializing detector to make it a bit more resilient +* Gitea pipelines have nocuda option for all architectures + +### 1.0.0-rc.85 +This is an UNSTABLE release. + +* jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image +* jfjoch_viewer: Handle spot count better in dataset info plots +* jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled + +### 1.0.0-rc.84 +This is an UNSTABLE release. + +* jfjoch_broker: Write in log which detector is being initialized +* Changes to automated build system + +### 1.0.0-rc.83 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before) +* CI: Fix script to generate python client + +### 1.0.0-rc.82 +This is an UNSTABLE release. + +* jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment) +* Frontend: Minor fixes +* Build scripts: Minor fixes to FFTW + +### 1.0.0-rc.81 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing +* jfjoch_broker: Add FFTW based indexer for CPU only indexing +* jfjoch_broker: Enable saving X-ray fluorescence spectra +* jfjoch_writer: Write total spot count (before filtering) +* jfjoch_viewer: Add more information on source, sample, and buttom to show ice rings +* jfjoch_viewer: Enable data processing inside the viewer + +CI: Moving from Gitlab to Gitea at PSI + +### 1.0.0-rc.80 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer + +### 1.0.0-rc.79 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance +* jfjoch_writer: Save nimages_per_trigger in detectorSpecific + +### 1.0.0-rc.78 +This is an UNSTABLE release. + +* jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore) + +### 1.0.0-rc.77 +This is an UNSTABLE release. + +* jfjoch_viewer: Display detector and dataset settings with tooltips +* jfjoch_viewer: Clean excessive HDF5 warnings +* jfjoch_viewer: Display unit cell +* jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset + +### 1.0.0-rc.76 +This is an UNSTABLE release. + +* jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones +* jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median) +* jfjoch_writer: Save space group number (non-NXmx addition) in addition to name +* jfjoch_viewer: Fix the bug on reading space_group as string +* jfjoch_viewer: Add missing resolution labels on rings +* jfjoch_viewer: Remove Q value from the status bar + +### 1.0.0-rc.75 +This is an UNSTABLE release. + +* jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being + +### 1.0.0-rc.74 +This is an UNSTABLE release. + +* jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing) +* jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization + +### 1.0.0-rc.73 +This is an UNSTABLE release. + +* jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default + +### 1.0.0-rc.72 +This is an UNSTABLE release. +This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Refactor of indexing and geometry refinement code +* jfjoch_broker: Handle space group/centering in refinement code +* jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images +* jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion +* jfjoch_writer: Experimental saving of reflections +* jfjoch_writer: Save space group name as string +* jfjoch_viewer: Add profile radius and B-factor +* jfjoch_viewer: Show 4 digits for wavelength +* jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings) +* FPGA: Use UDP destination port to distinguish between detector modules and data streams +* FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up) +* FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene) + +### 1.0.0-rc.71 +This is an UNSTABLE release. + +* jfjoch_broker: Remove resolution estimation via machine learning +* jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers +* jfjoch_viewer: Fix error when HDF5 files with indexing results couldn't be loaded on a machine without GPU + +### 1.0.0-rc.70 +This is an UNSTABLE release. +This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement +* jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards) +* jfjoch_broker: Enable geometry refinement - first try (work in progress) +* jfjoch_viewer: Fix deadlock when opening HTTP connections +* jfjoch_viewer: Display rings as ellipses with detector tilt +* jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image +* jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3) + +* Add Google Ceres a non-linear least-square optimization library to Jungfraujoch +* Add experimental detector calibration routines (for LaB6) +* Improve documentation on the ZeroMQ writer notification socket and detector geometry + +### 1.0.0-rc.69 +This is an UNSTABLE release. + +* jfjoch_viewer: Metadata can be modified for an open dataset (no option to save) +* jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments +* jfjoch_viewer: Resolution rings not enabled by default +* jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file + +### 1.0.0-rc.68 +This is an UNSTABLE release. + +* jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector +* jfjoch_broker: Default detector settings can be configured for each detector separately +* jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings +* jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing +* jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter +* jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI + +### 1.0.0-rc.67 +This is an UNSTABLE release. + +* jfjoch_broker: Enable SSL +* jfjoch_broker: Wilson B-factor only provided is fit is relatively OK (R^2 > 0.3); this will be refined much more in the future + +### 1.0.0-rc.66 +This is an UNSTABLE release. + +* jfjoch_broker: Indexers operate as thread pool, which is operating +* jfjoch_viewer: Increase interval between loading images + fix too many verbose messages + +### 1.0.0-rc.65 +This is an UNSTABLE release. + +* jfjoch_broker: Print information regarding used image pushers +* jfjoch_viewer: Allow syncing with Jungfraujoch server +* OpenAPI: Clarify licensing terms in the file + +### 1.0.0-rc.64 +This is an UNSTABLE release. + +* jfjoch_broker: Fix issue in receiver light with very long preparation time for threads +* jfjoch_broker: Add verbose option +* jfjoch_broker: Don't trigger pedestal if critical settings are not changed when loading detector settings +* jfjoch_broker: Detector left in busy state when detector settings were improper +* jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times +* jfjoch_lite_perf_test: Add verbose option + +### 1.0.0-rc.63 +This is an UNSTABLE release. + +* jfjoch_broker: Save NX/NY for grid scan result +* jfjoch_broker: Add processing time to CBOR output and plot +* jfjoch_writer: Add processing time to data file + +### 1.0.0-rc.62 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where low resolution spots were not counted properly +* jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count +* jfjoch_broker: Add more spot count information to CBOR +* jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles + +### 1.0.0-rc.61 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector +* jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results +* jfjoch_broker: High resolution limit for low res. spot counting can be adjusted + +### 1.0.0-rc.60 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released +* jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices +* jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength + +### 1.0.0-rc.59 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock. +* jfjoch_writer: Verbose option for debugging. + +### 1.0.0-rc.58 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix memory leak +* jfjoch_writer: Add detector_number/serial_number to master file + +### 1.0.0-rc.57 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible +* jfjoch_viewer: "Movie" mode + +### 1.0.0-rc.56 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation + +### 1.0.0-rc.55 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation + +### 1.0.0-rc.54 +This is an UNSTABLE release. + +* jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not) +* jfjoch_broker: Don't remove shared memory area when deactivating detector +* jfjoch_writer: Save writer release +* jfjoch_viewer: Increase time for the messages in the status bar + +### 1.0.0-rc.53 +This is an UNSTABLE release. + +* PCIe driver: Imperfect solution for RHEL 9.5+ changes +* jfjoch_writer: Fix to angle containers for AutoProc compatibility +* jfjoch_fpga_test: Use consecutive number for devices, not interleaved + +### 1.0.0-rc.52 +This is an UNSTABLE release. + +* jfjoch_viewer: Use warmer colors to distinguish from AareGUI +* jfjoch_viewer: Minor adjustments to DBus setting image number +* jfjoch_broker: Fix in low resolution spot count plotting + +### 1.0.0-rc.51 +This is an UNSTABLE release. + +* jfjoch_broker: Send preview in PNG format +* jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range +* jfjoch_broker: Provide ML resolution estimation in scan result +* jfjoch_broker: Allow removing beam center in web preview + +### 1.0.0-rc.50 +This is an UNSTABLE release. + +* The release fixes some of many bugs introduced in recent releases +* jfjoch_viewer: display predictions for indexed cells + +### 1.0.0-rc.49 +This is an UNSTABLE release. + +* jfjoch_broker: Handle sample temperature (K) and ring current (mA) to metadata +* jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current + +### 1.0.0-rc.48 +This is an UNSTABLE release. + +* jfjoch_broker: fix the bug when a unit cell was not exported for a scan result. + +### 1.0.0-rc.47 +This is an UNSTABLE release. + +* jfjoch_viewer: fix dbus service path +* jfjoch_writer: fix CBF/TIFF writing + +### 1.0.0-rc.46 +This is an UNSTABLE release. + +* jfjoch_viewer: remove dependency on image analysis + +### 1.0.0-rc.45 +This is an UNSTABLE release. + +* jfjoch_broker: Detector list returns pixel size (mm) + +### 1.0.0-rc.44 +This is an UNSTABLE release. + +* jfjoch_broker: more general definition of scan result export + +Braking changes: +* It removes additions to OpenAPI from 1.0.0-rc.43 +* It makes changes to the "unit_cell" definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations. + +### 1.0.0-rc.43 +This is an UNSTABLE release. + +* jfjoch_broker: Export grid scan results into a single data structure + +### 1.0.0-rc.42 +This is an UNSTABLE release. + +* jfjoch_broker: Add pixel_sum to CBOR output. +* jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine +* jfjoch_writer: Save pixel_sum + +### 1.0.0-rc.41 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction +* jfjoch_broker: Export GPU count via OpenAPI +* jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector + +* FPGA: Increase the number of integration bins to 2048 + +### 1.0.0-rc.40 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures +* jfjoch_broker: Improve metadata for rotation data collection +* jfjoch_broker: Better handling of plotting +* jfjoch_broker: FFT based indexing +* jfjoch_broker: Integration, first try, results not saved at the moment +* jfjoch_broker: Internal improvements in image handling + +* jfjoch_writer: Multiple adjustments adapt to changes in this release for new features +* jfjoch_writer: New state management model to improve clarity of error reporting + +* jfjoch_viewer: Remote control via DBus + +* Frontend: Multiple adjustments for new features +* Frontend: Grid scan plots + +WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken. + +### 1.0.0-rc.39 +* FPGA: Bugfix for pixel masked for data analysis if summation was on +* jfjoch_viewer: Fix segmentation fault when cursor was outside of image + +### 1.0.0-rc.38 +* jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via RES +* jfjoch_broker: Neural net model can use all 4 quadrants of the detector +* jfjoch_broker: For EIGER image time can be provided through /start +* jfjoch_viewer: Add image list option +* jfjoch_viewer: Drawing circular ROIs with shift +* jfjoch_viewer: Enable image summation +* jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer + +### 1.0.0-rc.37 +* jfjoch_broker: Make locking rules more flexible +* jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors +* jfjoch_viewer: Add status bar + +### 1.0.0-rc.36 +This is UNSTABLE release. Wait for new version to use in a production environment. + +* jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code) +* jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation) +* jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data +* jfjoch_writer: Increase buffering to 1000 images in the receiver +* jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5 + +### 1.0.0-rc.35 +This is UNSTABLE release, not properly tested. Wait for new version for using production. + +* jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting. +* jfjoch_writer: Save EIGER energy threshold +* jfjoch_writer: Add `/entry/sample/goniometer` for compatibility with `eiger2cbf` program + +### 1.0.0-rc.34 +This is UNSTABLE release - introducing new features, but not properly tested. Wait for new version for using production. + +* jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings) +* jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells +* jfjoch_broker: Handle situation when detector doesn't want to gracefully stop (to be tested) +* jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots +* jfjoch_viewer: Can extract data analysis results from "legacy" format +* jfjoch_viewer: Display dataset name +* FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels +* FPGA: Add pixel sum to module statistics +* FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI +* FPGA: Spot finder is back to full dynamic range (24-bit) +* FPGA: More debug features for internal FIFOs + +Known issues: +* ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts. +* EIGER data acquisition has an issue that is currently debugged + +### 1.0.0-rc.33 +* jfjoch_broker: Fix issue with EIGER settings being loaded improperly + +### 1.0.0-rc.32 +* jfjoch_broker: Refactor code for azimuthal integration for further improvements +* jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later) +* jfjoch_writer: Fix too much verbose information +* FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20) + +### 1.0.0-rc.31 +This is UNSTABLE release - introducing many features, but still needs more testing. +Expecting soon to put bugfix release. + +* jfjoch_writer: Allow to enable overwriting existing files (not enabled by default) +* jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default) +* jfjoch_viewer: Image viewer work early test version +* jfjoch_broker: Fixes to counting packets per dataset/image +* jfjoch_broker: Image buffer is accessible for outside to check images +* jfjoch_broker: error/saturated pixels and dedicated ROI "beam" can be tracked online +* jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU +* jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation +* jfjoch_broker: Fix bug in EIGER initializing +* jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot +* PCIe driver: Add PCIe link speed and width +* FPGA: Improve counting error/saturated/min/max pixels +* FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas +* FPGA: Require Vivado 2022.2 + +Warning: +There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency. + +### 1.0.0-rc.30 +* jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to "loose" images within the writer + +### 1.0.0-rc.29 +* jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU +* jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file +* jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info) +* jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly + +Warning - two breaking changes to OpenAPI: +* Handling of ROIs is through `/config/roi` path only for both circle and box ROIs, path in `/roi` are no longer accessible +* `broker_status` structure introduced in 1.0.0-rc.28 has member `message` and not `error_message` to allow +handling info/warning messages as well + +### 1.0.0-rc.28 +* jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI +* jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not complaint to OpenAPI specs +* jfjoch_test: remove header that failed when CUDA is absent during compilation +* frontend: add soft trigger button in data collection tab +* frontend: show error message when in error state +* CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF) + +### 1.0.0-rc.27 +* jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation +* jfjoch_broker: update pistache web server version +* jfjoch_writer: minor changes to republish logic +* Improvements to documentation + +### 1.0.0-rc.26 +* jfjoch_broker: implement ZeroMQ stream for image metadata information +* jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent +* jfjoch_broker: add crystal lattice plots +* jfjoch_broker: remove empty bins from the plots +* jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU "long" summation +* jfjoch_broker: Fix bug when mean background estimation / indexing rate where affected by previous experiment +* jfjoch_writer: fix missing "-w" parameter +* jfjoch_writer: temporary files have ".tmp" suffix +* jfjoch_writer: refactor logic for watermarks +* jfjoch_writer: report on internal FIFO utilization +* jfjoch_writer: clean-up naming for azimuthal integration and background estimate +* jfjoch_writer: write final background estimate and indexing rate in the master file +* tools/: remove unnecessary tools, make naming consistent +* CBOR: Add indexing rate and background estimate to end message +* CBOR: Clean-up documentation + +### 1.0.0-rc.25 + +* Updates to documentation +* License set to GPLv3 / OHL-S +* Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms +* Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5 + +### 1.0.0-rc.24 + +New FPGA functionality: +* EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit "real" depth is 23-bit + 1-bit signed) +* Output possible to 8, 16 and 32-bit data +* Threshold is applied before summation +* Pixel mask can be applied on FPGA +* Mark pixels with ADC content = 0 as bad pixels +* FPGA stores semantic version information (access via /sys/class/misc/jfjoch.../version) + +New software functionality: +* Long summation (above 256 frames) done on CPU +* Mechanism to save arbitrary data to HDF5 file +* ZeroMQ preview has option to send start message +* Rework pixel mask + add statistics displayed in web interface + +Bug fixes: +* Web frontend: Update preview image automatically during data acquisition +* jfjoch_broker: Error handling if CUDA driver is not installed +* jfjoch_broker: Correctly update progress during pedestal +* jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF +* jfjoch_action_test: enable HLS simulation + +Documentation improvement and placement in a dedicated directory diff --git a/_sources/CPU_DATA_ANALYSIS.md.txt b/_sources/CPU_DATA_ANALYSIS.md.txt new file mode 100644 index 00000000..a5c605fb --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS.md.txt @@ -0,0 +1,965 @@ +# CPU-side crystallographic data analysis (Jungfraujoch) + +This document describes the crystallographic algorithms implemented in Jungfraujoch for **CPU**- and **GPU**-side real‑time and near‑real‑time data analysis. + +**Scope.** The pipeline covered here comprises: + +1. geometry mapping and corrections, +2. azimuthal integration (powder/radial profiles), +3. Bragg spot finding (strong pixels → connected components → spot descriptors), +4. indexing (still and rotation modes), +5. Bravais lattice / centering inference, +6. geometry and lattice refinement, +7. reflection prediction (still and rotation), +8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free), +9. scaling and merging, +10. merge-level error modelling, outlier rejection and the resolution cutoff, +11. space-group determination from the merged intensities (Laue group, screw axes, centering) and the twinning check, +12. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref), +13. amplitude estimation (French–Wilson) and R-free test-set flagging, +14. optional model-based validation: R-free against a supplied model and 2Fo−Fc / Fo−Fc electron-density maps. + +## References + +The methods draw on, and in places reimplement, solutions from: + +- W. Kabsch, “XDS”, *Acta Cryst.* **D66** (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts). +- W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, *Acta Cryst.* **D66** (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors). +- T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts). +- J. Kieffer & J. P. Wright, "PyFAI: a Python library for high performance azimuthal integration on GPU", *Powder Diffraction* **28** (2013), S339-S350 (detector geometry definition, azimuthal integration) +- H. Powell, "The Rossmann Fourier autoindexing algorithm in MOSFLM", *Acta Cryst.* **D55** (1999), 1690-1695 (FFT indexing) +- S. French & K. Wilson, "On the treatment of negative intensity observations", *Acta Cryst.* **A34** (1978), 517-525 (Bayesian amplitude estimation from intensities). +- A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free cross-validation). +- M. Wojdyr, "GEMMI: A library for structural biology", *J. Open Source Softw.* **7** (2022), 4200 (model / structure-factor / map machinery used in §14). +- J. P. Wright, "Experiences with GPU decompression for bitshuffle + LZ4 data", HDF5 User Group meeting (2021), and [github.com/jonwright/bslz4decoders](https://github.com/jonwright/bslz4decoders) (device-side decoding of bitshuffle+LZ4 images, §0). +- Z. Otwinowski & W. Minor, "Processing of X-ray diffraction data collected in oscillation mode", *Methods Enzymol.* **276** (1997), 307-326 (reweighted, de-biased profile-fit variances). +- G. Winter et al., "DIALS: implementation and evaluation of a new integration package", *Acta Cryst.* **D74** (2018), 85-97, and J. Beilsten-Edmands et al., *Acta Cryst.* **D76** (2020), 385-399 (CC1/2 resolution cutoff, merge outlier rejection, scaling error model). +- P. Evans, "Scaling and assessment of data quality", *Acta Cryst.* **D62** (2006), 72-82, and P. R. Evans, *Acta Cryst.* **D67** (2011), 282-292 (POINTLESS: operator-by-operator point-group scoring, and the axial-zone screw-absence test). +- A. G. W. Leslie & H. R. Powell, "Processing diffraction data with MOSFLM" (2007), NATO Science Series II **245**, 41-51 (post-refinement practice: what is refined per image and what over a wedge). +- K. Diederichs & P. A. Karplus, *Nat. Struct. Biol.* **4** (1997), 269-275, and P. A. Karplus & K. Diederichs, *Science* **336** (2012), 1030-1033 (R_meas / R_pim, CC1/2 and CC\*). +- IUCr Commission on Crystallographic Nomenclature, "Statistical descriptors in crystallography", *Acta Cryst.* **A45** (1989), 63-75, and *Acta Cryst.* **A51** (1995), 565-569 (uncertainty conventions). + +(list is not exhaustive; the full citations, with DOIs, are in [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md)) + +## 0. Getting the image onto the GPU: device-side bitshuffle+LZ4 decoding + +Images arrive bitshuffle+LZ4 compressed (HDF5 filter 32008), and everything from §1 onwards runs on +the GPU when one is present. Instead of decompressing on the host and uploading the image, the +compressed chunk is uploaded — a few MB rather than tens of MB — and decoded on the device. The +approach follows Jon Wright (ESRF); the kernels are Jungfraujoch's own. + +Two kernels do the work: + +1. **LZ4, one warp per bitshuffle block.** Blocks are independent, so the parallelism is across + them; within a warp every lane runs the same sequence parser over the same bytes, and the + literal and match copies are split across the 32 lanes so the stores coalesce. An overlapping + match is treated as a pattern of period `offset` sourced from bytes that already precede the + write position, which keeps it parallel rather than a serial byte loop; `offset == 1` (a run of + one repeated byte, the common case in sparse detector data) and power-of-two offsets avoid the + modulo altogether. Because the lanes cooperate on the copies, each one is followed by + `__syncwarp()` — a later match can read bytes another lane wrote, and since Volta that ordering + is not implicit. +2. **The bitshuffle inverse fused with preprocessing.** One thread owns one group of 8 elements + across every byte-plane, so once it has transposed its 8 bytes out of each plane it holds 8 + complete elements — and it applies the pixel mask, the error marker and the saturation cap and + emits 8 finished `int32` pixels directly. The decompressed image is therefore never materialised + in device memory at all, which removes a frame-sized buffer per worker and a full-frame write + plus read from the pipeline. Staging nothing in shared memory also means the kernel has no + dynamic-shared-memory request, so it is indifferent to the bitshuffle block size the file + declares. For 8-bit images there is a single plane and the assembly degenerates to a copy. + +The block offsets inside the container can only be discovered by reading the block lengths in +order, so that scan stays on the host. + +Only `BSHUF_LZ4` is decoded on the device. For the zstd variants (`BSHUF_ZSTD`, `BSHUF_ZSTD_RLE`, +`BSHUF_ZSTD_RLE_HUFF`), and for uncompressed or float images, `BSLZ4DecoderGPU::Supports()` returns +false and the pipeline decompresses on the host and uploads as before. + +The container arrives off the network or off disk and is not trusted. Everything checkable on the +host — declared sizes, the block scan, a block size that is not a multiple of 8 elements, a block +count the chunk could not hold, trailing bytes — is rejected before any work is queued; the kernel +additionally flags a block that did not decode to exactly its declared length, which becomes an +exception once the caller has synchronised. That last check matters because the decode buffers are +reused frame to frame: a block that stopped early would leave the *previous* image's most significant +byte-plane in place, which reads not as a missing corner but as real pixels several powers of two too +bright. + +## 1. Geometry, reciprocal-space mapping, and basic quantities + +### 1.1 Coordinate conventions + +For a pixel coordinate $(x,y)$ (in pixels), Jungfraujoch converts to a laboratory direction vector via: + +1. shift by direct-beam position $(x_\mathrm{beam}, y_\mathrm{beam})$, +2. scale by pixel size $p$ (mm), +3. set detector distance $D$ (mm), +4. apply detector orientation rotation $R_\mathrm{det}$ (PyFAI-like parameterization). + +The unnormalized detector coordinate (mm) is: +$ +\mathbf{r}_\mathrm{det}(x,y) = +\begin{pmatrix} +(x-x_\mathrm{beam})p\\ +(y-y_\mathrm{beam})p\\ +D +\end{pmatrix}. +$ + +The lab-frame vector is: +$ +\mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. +$ + +Let the incident wavevector magnitude be $k = 1/\lambda$ in Å$^{-1}$, and define: +$ +\mathbf{S}_0 = (0,0,k). +$ + +The **reciprocal-space scattering vector** associated with pixel $(x,y)$ is: +$ +\mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. +$ + +This $\mathbf{s}$ is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement. + +### 1.2 Two-theta, azimuth, resolution and $q$ + +The scattering angle $2\theta$ is computed from $\mathbf{r}_\mathrm{lab}$ via: +$ +2\theta = \arctan\!\left(\frac{\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2}}{z_\mathrm{lab}}\right). +$ + +Resolution (Å) at a pixel is: +$ +d = \frac{\lambda}{2\sin\theta}. +$ + +The magnitude $q = 2\pi/d$ is used for radial binning and ice-ring handling. + +### 1.3 Distance from the Ewald sphere + +For a reciprocal lattice point $\mathbf{p}$ (Å$^{-1}$), define: +$ +\Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. +$ +Jungfraujoch uses $|\Delta_\mathrm{Ewald}|$ as an operational proxy for excitation error. This appears in: +- still prediction (accept if $|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}$), +- profile radius estimation (see §11.1), +- still partiality option in scaling/merging (§10.2). + +### 1.4 Measuring the direct beam before indexing + +The beam centre in the file is often a placeholder, and nothing else measures it until +post-refinement (§7.5) — by which time a wrong centre has already chosen the lattice. With +`--estimate-beam-center` it is measured from spot positions alone, before anything is indexed. Two +exact facts about a rotation sweep supply the two coordinates: + +**Friedel mates half a turn apart.** The Laue condition fixes the component of $\mathbf{q}$ along the +beam, $q_\parallel = -\lVert\mathbf{q}\rVert^2\lambda/2$. Rotating 180° about the spindle $\mathbf{m}$ +negates the two components perpendicular to $\mathbf{m}$ and taking $-h$ negates all three, so +together they negate **only** the component along $\mathbf{m}$ and leave $q_\parallel$ untouched. With +the spindle perpendicular to the beam, $-h$ therefore diffracts at $\varphi+180°$ exactly where $h$ +diffracts at $\varphi$, and its spot sits at the mirror image of $h$'s along the spindle. This gives +the beam coordinate **along** the spindle. Only the reciprocal lattice's centrosymmetry is needed for +the geometry; Friedel's law $|F(h)|=|F(-h)|$ is used separately, to tell a true pairing from an +accidental one. + +**The second crossing.** The same reflection meets the Ewald sphere twice, at two angles that are +generally *not* 180° apart, differing only in the sign of the lab component perpendicular to both +$\mathbf{m}$ and the beam. This gives the remaining coordinate. The two crossings are separated by a +sweep angle fixed by the reflection's own position, which is what identifies genuine pairs. + +Neither observable requires a cell or an orientation matrix: each candidate pairing votes for a beam +coordinate, and the true value accumulates while wrong pairings scatter. A Friedel pair needs both +$\varphi$ and $\varphi+180°$ recorded, so a sweep of $S°$ yields only $S-180$ degrees' worth of pairs +— a sweep of exactly half a turn yields none, and the estimator is refused below a floor on that span. + +The mirror is exact in the **laboratory** frame, so it is sensitive to the spindle direction. A skew +of the spindle about the beam *spreads* the vote rather than shifting it, and is fitted alongside the +centre (`--no-fit-spindle` keeps the axis from the file); a tilt of the spindle towards the beam is +measured and reported but not applied, being confounded with the detector tilt until that is fitted +too. Nothing inside the fit can tell that the vote settled on the wrong periodic maximum — every +frame pair agrees with every other — so the answer is accepted only if it does not move when the +search is started from a different position. Frames are sampled away from both ends of the sweep, +where shutter synchronisation can spoil an image. + +Where the sweep is shorter than half a turn the spot symmetry cannot be formed, and the centre is +taken instead from the centroid of the radial background profile, which needs only a few images. +Where neither method can measure the centre, the value from the file is kept. + +--- + +## 2. Azimuthal integration (radial profiles) + +Azimuthal integration produces a radial profile $I(q)$ or $I(d)$ by histogramming pixels into radial bins. Pixels are **not split** across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 $\phi$ sectors (`azim_bins`, `--azim-phi-bins`), giving a **2D $q\times\phi$ profile** that exposes azimuthal anisotropy such as detector shadowing or sample texture. + +### 2.1 Histogram estimator + +Let bin index $b(x,y)$ be precomputed from $q(x,y)$ (or equivalently from $d(x,y)$) and, when $\phi$ sectors are enabled, the azimuth $\phi(x,y)$ — so $b = b_q + b_\phi B_q$. For each bin $b$: + +- accumulate corrected intensity and its square: + $ + S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad + S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, + $ +- and count: + $ + N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. + $ + +The profile reports both the mean $\bar{I}_b = S_b / N_b$ (when $N_b>0$) and a per-bin sample standard deviation $\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}$ (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded. + +### 2.2 Corrections applied + +Two standard corrections are available: + +**(i) Solid angle / geometric correction.** A flat pixel's solid angle falls off with the **incidence angle $\alpha$ between the scattered ray and the detector normal**. With the in-plane detector offsets $u=(x-x_\mathrm{beam})p$ and $v=(y-y_\mathrm{beam})p$ (§1.1) and detector distance $D$, +$ +\cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad +C_\Omega = \cos^3\alpha, +$ +applied — like the polarization term below — as a **divisor** (intensities are scaled by $1/\cos^3\alpha$), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because $\alpha$ is evaluated in the detector's own frame it is **invariant under detector tilt** ($\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}$), matching PyFAI's `solidAngleArray` and MAX IV azint. It reduces to the commonly quoted $\cos^3(2\theta)$ form only for an untilted detector, where the incidence angle coincides with the scattering angle. + +**(ii) Polarization correction.** With polarization coefficient $P$ (beamline dependent) and azimuth $\phi$: +$ +C_\mathrm{pol}(2\theta,\phi) = +\frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), +$ +applied as a divisor to intensities (i.e. scale by $1/C_\mathrm{pol}$) when enabled. + +### 2.3 Background estimate for profiles + +A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid $Q$ window (default $2\pi/5$ to $2\pi/3$ Å$^{-1}$). This background is used for monitoring and diagnostics; it is **not** the same as the local Bragg-spot background used in summation integration (§9.2). + +--- + +## 3. Spot finding (strong pixels → Bragg spots) + +Spot finding is a two-stage process: + +1. **Strong-pixel selection** using intensity and/or local signal-to-noise criteria. +2. **Connected-component labeling (CCL)** to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction. + +### 3.1 Strong-pixel detection by local statistics + +For each pixel $i$ with value $v_i$, consider a square window (nominally $31\times 31$ pixels) around it. Let the window contain $n$ valid pixels (excluding masked/bad/saturated), and define: +$ +\Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. +$ + +To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: +$ +\Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. +$ + +A variance-like quantity proportional to $n'^2$ is formed: +$ +V = n'\Sigma_2' - (\Sigma')^2, +$ +and the deviation-from-mean quantity: +$ +\Delta = v_i n' - \Sigma'. +$ + +A pixel is considered strong if: +- it is above a photon/count threshold, and +- its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and +- $\Delta>0$, and +- the squared deviation exceeds a scaled variance: + $ + \Delta^2 > V\cdot T^2, + $ + where $T$ is the configured signal-to-noise threshold. + +This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast. + +The test is applied in **two passes** over the image. The first is as described above. The second repeats it with every pixel found strong by the first excluded from the local background — it is treated exactly like a saturated pixel, so it contributes to no window it falls into and stays strong itself. This matters for any spot wide enough to reach into its own background box: on a single pass such a spot inflates the mean and variance it is then tested against, and its outer pixels fail the criterion. Excluding the core recovers them, so the spot is reported with its true extent rather than its brightest few pixels. Both the CPU and GPU implementations run these two passes and return the same spot list for the same frame. + +Special cases: +- saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots), +- invalid pixels are never strong. + +### 3.2 Adaptive (self-calibrating) detection + +The local-statistics test above needs a fixed photon/count threshold whose correct value depends on the background level, which varies between datasets. The **adaptive** mode (`--adaptive-spots`; the default in `rugnux` and in the viewer for both stills and rotation data, `--no-adaptive-spots` reverts) derives that threshold from each image's own noise, per resolution ring, so no per-dataset value is needed. It admits more spots than the fixed threshold, including genuine reflections that belong to no indexed lattice; these are down-weighted rather than filtered in the per-image geometry fit (§7.4). + +Pixels are binned into the same resolution rings as the azimuthal integrator (§2). For each ring a robust background is estimated in three passes: one plain pass over all valid pixels, then two $\sigma$-clipping passes that keep only pixels within $\pm 3\sigma$ of the current ring mean (removing the Bragg peaks from the background estimate). This yields a per-ring background mean $\mu_b$ and scatter $\sigma_b$. + +The ring's detection threshold is the larger of two arms, +$ +t_b = \max\!\big(\;\mu_b + z\,\sqrt{\sigma_b^2 + \sigma_\mathrm{read}^2}\;,\;\; k_\mathrm{Poisson}(\mu_b, p)\;\big), +$ +where $k_\mathrm{Poisson}(\mu_b,p)$ is the smallest count whose Poisson$(\mu_b)$ upper tail is $\le p$. The Poisson arm is correct where the background is countable (a bright low-resolution ring gets a high threshold); the Gaussian arm — floored by a detector-level excess-noise constant $\sigma_\mathrm{read}$ — takes over on near-empty high-resolution rings, where the Poisson arm degenerates to "one photon is significant" and would flood. The operating point $p = E/N$ is set from a single portable knob $E$, the expected number of false pixels tolerated per frame (`--spot-false-pixels`, default 100), with $N$ the number of valid pixels. Because $p$ and every $\mu_b,\sigma_b$ come from the image itself, the same $E$ lands a sensible photon threshold on strong and weak datasets alike, with no per-dataset tuning. Rings too sparse to characterise (fewer than ~40 pixels) fall back to a whole-frame background. A pixel is strong when $v_i \ge t_b$ for its ring (saturated pixels are still forced strong); the strong pixels then feed the same CCL stage (§3.4). The signal-to-noise and photon-count criteria of §3.1 are not used in this mode. + +Because detection reads the pixel's ring, a pixel that falls outside the azimuthal-integration $q$ range has no ring and can never be strong: the integration range bounds what adaptive detection can see. Both upper limits are therefore optional and default to the detector itself — the azimuthal integration runs to the highest $q$ any pixel of the detector reaches (`--azim-max-q` unset), and spot finding is not clipped in resolution (`--spot-high-resolution` unset), for rotation data as well as stills. Setting either one narrows detection accordingly — appropriate for weak, high-background data, where the spots admitted at the detector edge are dominated by noise. + +**Fused GPU engine.** The per-ring reduction the adaptive threshold needs is the *same* reduction the azimuthal integrator performs. On the GPU path the two are fused into a single image pass (`AdaptiveSpotFinderGPU`): one reduction accumulates the corrected per-ring sums for the azimuthal profile (§2) *and* the raw per-ring statistics for the threshold, after which a light kernel flags the strong pixels. One GPU pass therefore replaces both the separate azimuthal-integration pass and the host-side adaptive spot-finding pass, at a small fraction of the CPU finder's cost per frame and producing the same spot list and azimuthal profile. It is enabled by default in the offline `rugnux` path, the interactive viewer and the online receiver. + +**Online.** `spot_finding_settings` in the REST API carries `adaptive_threshold` and `false_pixels_per_frame`, so the mode is reachable from the broker and from the web frontend as well as from `rugnux` and the viewer. It defaults to *off* online, unlike `rugnux` and the viewer, because the broker serves both workflows and only one of them can run it: spots are found in software only on the DECTRIS/SIMPLON path, while the JUNGFRAU and EIGER workflows find them on the FPGA at its own fixed threshold. Setting `adaptive_threshold` on those is refused with an error rather than accepted and ignored, so a detection setting that had no effect cannot be mistaken for one that did. + +### 3.3 Resolution and ice-ring handling + +Spot finding can be restricted to a resolution range $[d_\mathrm{high}, d_\mathrm{low}]$ by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6). + +A single per-image **ice-ring score** is derived from a radial profile: for each hexagonal-ice powder ring (positions $d$ from Moreau *et al.*, Acta Cryst D77, 2021), the profile intensity at the ring is divided by a smooth background estimated from the *whole* profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring's ratio is reported (1 = no ice, $>1$ = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring's own edge (offline processing defaults to a fine 0.01 1/Å spacing, `--azim-q-spacing`, so the rings are well resolved). The reported quantity is the ice *magnitude* rather than a significance: with many photons any real ice ring is statistically significant, so significance does not discriminate. + +The profile the score is read off is the **peak-excluded** one, not the plain azimuthal integration: where adaptive spot finding runs (§3.2 — the offline and viewer default), the score uses the sigma-clipped per-resolution-ring background that finder already computes for its threshold. This matters more than it sounds. A plain azimuthal profile is a per-ring *mean*, so a few strong low-resolution reflections landing in a ring's bin raise it exactly as ice would; measured over a rotation battery, that alone ranked ice-free crystals above crystals that really are iced. An ice ring is azimuthally smooth and survives the sigma clip, while Bragg peaks do not, so on the clipped profile ice-free crystals sit near 1 and crystals with confirmed ice above 2. Only where no adaptive finder ran (the FPGA workflow) does the score fall back to the plain profile. + +The radial profile sees ice only as a **smooth powder ring**. Ice in large crystallites diffracts as discrete spots, leaves the profile flat, and is invisible to the score above, so a second channel is read from the spot list itself: the spots found on the ice bands are counted against the spots found in the ice-free flanks $[w,2w)$ either side of each band, rescaled to the bands' own $q$ width (a flank landing on another ring is dropped with its width). The indicator is the ratio pooled over the run — per image the control is a handful of spots and the ratio means nothing — and it is taken before the spot-count filter, which orders ice spots last and would discard them first. The two channels barely overlap: smooth ice reads high on the profile and ~1 on the spots, textured ice the reverse, and a clean crystal ~1 on both. Both counts are stored per image (`spot_count_ice_rings`, `spot_count_ice_control`; HDF5 `/entry/MX/peakCountIceRingRes`, `/entry/MX/peakCountIceRingControl`). + +Both channels are used offline as a **gate** on ice handling: unless the run reaches `--ice-min-score` (default 1.5) on the profile or `--ice-min-spot-ratio` (default 2.0) on the spots — 0 disables a channel — ice-ring flagging and the exclusion from the scale fit (§10.10) are skipped. The eleven fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. + +A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in $1/d$ (or $q$) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low. + +### 3.4 Connected-component labeling (CCL) + +Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with: + +- centroid $(x,y)$ (often intensity-weighted), +- pixel count (spot size), +- integrated spot intensity proxy (sum of pixel values), +- resolution $d$ at the centroid (or mean over pixels), +- and quality flags (e.g. ice-ring classification). + +Spot-level filters include minimum/maximum pixel count and resolution limits. + +The host implementation (`StrongPixelSet::sparseccl`) is the SparseCCL of the ACTS/traccc project: it +runs over the strong pixels sorted row-major, uses a sliding window over the previous line and a +union-find whose root is each component's lowest index. On the GPU the same labelling runs **on the +device** (`SpotExtractorGPU`): the packed strong-pixel bitmask is compacted into that same sorted +list without leaving the card, each pixel finds its at most four earlier 8-neighbours by binary +search, and a lock-free union-find with path halving labels them. Only the finished spot list — a few +hundred entries — comes back to the host, instead of the whole bitmask (2.26 MB per frame at 18 MP). +The two implementations produce the same components, in the same order, with the same pixel counts +and intensities; `tests/SpotExtractorGPUParityTest.cpp` holds them to it. The device version is also +insensitive to frame content: the host sliding window becomes quadratic when many pixels light up in +one detector line — a hot module, or a diffraction ring where it runs tangent to a row — which costs +tens to hundreds of milliseconds on such a frame, while the device version stays under a millisecond. + +### 3.5 Adaptive per-image minimum spot size + +The minimum-pixels-per-spot filter (§3.4) trades sensitivity against noise: a small value keeps faint one- or two-pixel spots — real signal on strong data, but detector noise on high-background frames — while a larger value keeps only well-formed spots. The best value is dataset-dependent, so for serial-stills indexing it can be chosen **per image** rather than fixed. The frame is indexed three times, at min-pix 3, 2 and 1, and the setting that maximises + +$$ \frac{n_\mathrm{indexed}^2}{n_\mathrm{total}} \quad\text{(indexed-spot count weighted by indexed fraction)} $$ + +is kept; the frame is then integrated once at that min-pix. The fraction factor discounts the extra spots a smaller min-pix admits *unless the lattice actually explains them*, so strong frames keep their real weak spots (extending resolution) while noise-flooded frames stay strict. Because min-pix filters the connected components *after* detection, strong-pixel detection AND the connected-component labelling both run **once** per frame, and the three attempts only repeat the spot-level filter; the azimuthal profile is the one that single detection pass computed. The winning attempt's spot list is kept rather than re-extracted, so the frame that is integrated is exactly the frame that was scored. This is a **stills-only, indexing-path** option — rotation indexing builds one global lattice from all frames and keeps a fixed min-pix. In `rugnux` it is the default; giving an explicit `--min-pix-per-spot` pins a fixed value instead. + +--- + +## 4. Indexing overview + +Indexing maps observed reciprocal-space vectors $\mathbf{s}_i$ to a lattice such that: +$ +\mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, +$ +with integer $(h_i,k_i,l_i)$. + +Jungfraujoch supports two complementary indexing strategies: + +1. **FFT-based indexing** (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples. +2. **Fast-feedback indexing** (TORO-like): requires an approximate unit cell; optimized for speed and feedback. + +Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return **more than one lattice** per image (multi-lattice indexing; see §5.4). + +### 4.1 Indexed-spot decision (inlier test) + +Given a trial lattice with direct basis vectors $\mathbf{a},\mathbf{b},\mathbf{c}$ (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: +$ +h_f = \mathbf{s}\cdot\mathbf{a},\quad +k_f = \mathbf{s}\cdot\mathbf{b},\quad +l_f = \mathbf{s}\cdot\mathbf{c}. +$ +Let $(h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))$ and define the fractional residual: +$ +\delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. +$ +A spot is indexed if $\delta^2 < \tau^2$, where $\tau$ is the configured tolerance. + +For indexed spots, the reciprocal lattice point $\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*$ is used to compute $\Delta_\mathrm{Ewald}(\mathbf{p})$ (stored as a diagnostic and later used in profile-radius estimation). + +--- + +## 5. FFT indexing (unknown unit cell) + +FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms. + +### 5.1 Directional projections and histograms + +Choose a set of unit vectors $\{\mathbf{u}_d\}$ on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction $d$, form a histogram in the scalar projection: +$ +t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. +$ + +Bin width is chosen approximately as: +$ +\Delta t \approx \frac{1}{2 L_\mathrm{max}}, +$ +where $L_\mathrm{max}$ is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum $q$ used (set by a high-resolution cutoff for indexing). + +### 5.2 FFT peak picking and candidate vectors + +For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along $\mathbf{u}_d$. Each direction yields a candidate real-space length $L$ chosen **not** by raw magnitude but by **maximum prominence above a running-mean local background** (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to $L\ge L_\mathrm{min}$. + +Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. + +A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak. + +### 5.3 Lattice reduction and cell candidates + +Triples of candidate vectors are combined to form candidate bases $(\mathbf{A},\mathbf{B},\mathbf{C})$, each reduced to its **Niggli-reduced cell** (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a **widened fallback** anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses. + +### 5.4 Robust refinement and best-cell selection + +Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked: +1. more indexed spots wins — **unless** two candidates index within ~10 % of each other, in which case +2. the **smaller-volume** cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then +3. the smaller refinement score, then the spot count again. + +Selection is **not limited to a single lattice**: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded. + +An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant). + +--- + +## 6. Bravais lattice / centering inference (“lattice search”) + +If the space group is supplied by the user, its lattice constraints are assumed for refinement and subsequent processing. + +If not, Jungfraujoch attempts to infer the most plausible Bravais lattice type from the metric tensor after Niggli reduction: + +1. **Niggli reduction** is performed to obtain a reduced cell in $G^6$ representation (Gruber vector). +2. The reduced cell is compared against a list of Niggli classes corresponding to Bravais lattices and centerings. +3. The highest-symmetry class that matches within tolerances is selected (relative metric tolerance and angular tolerance). + +The output includes: +- a conventional cell, +- crystal system (triclinic, monoclinic, …), +- centering symbol (one of $P, C, I, F, R$; the $A/B$ variants are not emitted here — they are handled only later as prediction absences, §8.4). + +This stage provides centering information used for systematic absences in prediction (§8.4) and for reporting. + +**A metric symmetry has to earn itself.** The class is chosen from the *unrefined* candidate against a fixed angular tolerance (3°), so a lattice that is pseudo-symmetric to a few tenths of a degree is promoted a class too far — and the constraint then snaps a real angle to the ideal one, which throws nearly every reflection of every frame outside tolerance. Measured on a monoclinic crystal pseudo-C-orthorhombic to 0.42°, the promoted cell indexed 2 of 60 frames where its own primitive cell indexed 39: the same lattice, $\mathbf{b}_{oC}=-(\mathbf{a}+2\mathbf{c})$, at exactly twice the volume. A more accurate candidate is more likely to be promoted, not less: a run escapes the promotion only when the raw candidate misses the tolerance window. + +For rotation data the first pass therefore refines the constrained cell *and* an unconstrained (triclinic) one from the same spots — which it finds itself, over a sample spread across the sweep, rather than reading what the acquisition wrote — and settles the two on how many of a fixed set of validation frames each actually indexes. The bar is a clear majority, not a margin. An unconstrained refinement holds no cell parameter fixed, so it can only index at least as many frames as the constrained one, and on genuine symmetry it indexes a few more — a percentage margin therefore demotes real lattices (measured: a genuine $I$-centred orthorhombic to $P1$). Only a constrained cell that fails outright while its unconstrained cell works distinguishes a false promotion. The intensities settle the space group later regardless (§13). + +**Note.** In ambiguous or special cases, forcing space group to $P1$ (no symmetry assumptions) is recommended. + +--- + +## 7. Geometry and lattice refinement + +Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices. + +### 7.1 Parameterization + +The refinement jointly optimizes, depending on mode and constraints: + +- beam center $(x_\mathrm{beam}, y_\mathrm{beam})$, +- detector distance $D$, +- detector tilt angles (two-angle model; third rotation often held at 0), +- rotation axis direction (for rotation datasets), +- crystal orientation (a global rotation), +- unit-cell parameters, with constraints determined by inferred crystal system. + +By default only the beam center, unit cell and crystal orientation are refined; the detector distance, tilt angles and rotation-axis direction are held fixed unless explicitly enabled. A lighter **orientation-only** mode refines just the crystal orientation, for stills whose geometry is already trusted. It carries a weak small-rotation prior penalising the whole angle-axis vector (all three components, at a low weight); what it is there for is the poorly-determined out-of-plane component, which is the one the data barely constrain. + +For higher symmetries, constraints are enforced, e.g. +- cubic: $a=b=c,\ \alpha=\beta=\gamma=90^\circ$, +- tetragonal: $a=b$, +- hexagonal: $a=b,\ \gamma=120^\circ$, +- monoclinic (unique axis $b$): $\alpha=\gamma=90^\circ$, $\beta$ refined. + +### 7.2 Residuals and objective + +For each indexed spot assigned integer $(h,k,l)$, compute: + +- observed reciprocal vector $\mathbf{s}_\mathrm{obs}$ from its detector position and current geometry, +- predicted reciprocal vector $\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})$. + +Residual is: +$ +\mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. +$ + +A non-linear least squares solver minimizes $\sum \|\mathbf{r}\|^2$ over all selected inlier spots. + +### 7.3 Rotation datasets: bringing observations to a common reference frame + +For oscillation/rotation data, each image corresponds to a rotation angle $\phi$ about an axis $\mathbf{m}_2$. Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: +$ +\mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, +$ +with $R(\phi)$ constructed from the axis-angle representation of the goniometer model. The angle $\phi$ is taken at the centre of each frame's oscillation (the frame angle plus half the oscillation width). + +### 7.4 Multi-stage tightening of inlier tolerance + +Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance $0.3\to0.2\to0.1$), which stabilizes convergence when starting from imperfect indexing and approximate geometry. + +The loose first stage necessarily admits some spots that are not reflections of this lattice — the fraction of *randomly* placed spots inside a fractional-Miller tolerance $t$ is $\tfrac{4}{3}\pi t^3$, i.e. 11 % at $t=0.3$ — and an unweighted fit lets them pull the orientation. Each residual is therefore weighted by how strong its spot is **for its resolution**: the frame's spots are cut into equal-count resolution shells and each intensity is divided by its shell median, mapped to $w^2=r/(1+r)$. The shell normalisation is what makes this safe — genuine high-resolution spots are legitimately weaker and carry the cell and distance information, so an un-normalised intensity weight would suppress exactly the spots the fit needs. The weight is a property of the spot and never of the current residual, so it does not depend on how far the geometry is from convergence. + +### 7.5 Rotation geometry post-refinement (two-pass) + +The refinement above (§7.2) runs per image against that image's spots. For rotation data an additional **post-refinement** (on by default; `--rotation-no-postrefine` disables it) improves the detector distance, beam centre and crystal cell/axis using **all** frames at once, then re-integrates: + +1. **Pass 1** integrates, scales and merges at the header geometry. +2. From pass-1's integrated reflections, the geometry is refined over all frames (Ceres, robust loss) in **two separate steps** rather than one joint fit: + - **Step A**: crystal cell scale + goniometer-axis direction, from the observed rotation angles (a distance-independent excitation residual). + - **Step B**: shared detector distance + beam centre, from the observed spot positions, with the cell held at step A — so the positional residual is no longer degenerate with the cell scale. + + Each step is **cross-validated** on a deterministic split of the *reflections* (an avalanche-mixed $hkl$ hash, not a frame split and not an $h+k+l$ parity, which would collide with a centering condition and leave the held-out half empty): fitted on one half, committed only if it lowers the held-out residual, otherwise left at nominal. The solver bounds the move — distance within ±5 %, beam centre within ±15 px — and detector tilt is held fixed, being gauge-coupled to the crystal orientation on a single crystal. +3. **Pass 2** re-indexes de novo and re-integrates at the committed geometry, reusing pass-1's space group for the merge only. Only the **detector distance and beam centre** carry over: the refined cell and axis are used to make step B well-posed, but pass 2 re-indexes from scratch, so they are not propagated. + +The refined pass is written as the canonical `_*` output; the pass-1 (header-geometry) result is kept alongside as `_01_*` for comparison. + +**Goniometer rotation scale (report only).** A stage that turns further than it was commanded to leaves no trace in the file, because the stored $\omega$ values *are* the commanded ones; the excess then presents as the crystal drifting, in this program and in others. Step A already measures it without a new degree of freedom: its residual rotates by $-\phi\,\mathbf{u}$ with $\mathbf{u}$ an **unnormalised** 3-vector, so $|\mathbf{u}|$ is the factor by which the stage actually turned, and normalising the axis throws it away. It is reported, and warned about beyond 0.5 %, under the same cross-validation that gates the cell move — a fold that merely soaked up noise cannot raise the flag. It is a detector, not a calibration: nothing corrects the data, and it **under-reads** the true magnitude, because the fit only sees reflections that indexed at the nominal angle and per-frame orientation refinement has already absorbed part of the error. + +### 7.6 Detector geometry from powder rings + +Everything above fits the geometry to *Bragg* data, where the beam centre is the weakest parameter: it is gauge-coupled to the crystal orientation, which is why §7.5 restrains it toward the header value and commits only a sub-1 % move. A **powder ring has no orientation to be coupled to**. Where it falls on the detector depends on the geometry and on nothing else, which makes a calibrant — LaB₆, silver behenate, CeO₂, silicon — or even ice an independent constraint on exactly the quantity Bragg data cannot pin. + +The ring positions are matched to the observed rings and the geometry is refined (Ceres, five parameters: beam centre, distance, and the two detector tilts) so that the $|s|$ predicted at each observed ring point matches the ring it belongs to. + +**Calibrants.** LaB₆, silver behenate, CeO₂ and silicon are held as unit cells and their rings enumerated from them. Ice is held as the eleven **measured** hexagonal-ice ring positions of §3.3 instead, because hexagonal ice is $P6_3/mmc$ and enumerating $hkl$ from its cell would emit rings that are systematically absent. A calibrant is therefore a list of ring $q$ values throughout, not a cell. + +**What a ring can and cannot determine.** A ring is a conic centred on the beam, so a wrong centre makes its apparent radius oscillate once per turn, $r(\phi)=R+\delta_x\cos\phi+\delta_y\sin\phi$, with the **same amplitude on every ring**. A detector tilt $\beta$ produces a $\cos\phi$ term too — not the $\cos2\phi$ one might expect — but one that grows as the ring's radius *squared*, $r(\phi)=R+(R^2/F)(\beta_x\cos\phi+\beta_y\sin\phi)$; the true $\cos2\phi$ term is $O(R^3\beta^2/F^2)$, hundredths of a pixel. The two are therefore separated by how the amplitude scales with radius, which needs **at least two rings** — on a single ring they are exactly degenerate. None of this uses the calibrant's $d$-spacings, so the centre is determined without assuming anything about the standard. + +The **distance** is different: it follows from $r=F\tan2\theta$ with $\sin\theta=\lambda/2d$, so a fractional error in the lattice constant passes straight into it, and the $\lambda$–$F$ pair is separated only by the curvature of $\tan(2\arcsin(\lambda/2d))$ across the rings — $\partial\ln r/\partial\ln F=1$ at every ring against $\partial\ln r/\partial\ln\lambda=4\tan\theta/\sin4\theta$, which runs from about 1.05 at low angle to 1.43 at high. That lever collapses as the detector moves back and the rings crowd into small $2\theta$, so distance is a short-distance measurement and the wavelength is better calibrated by other means. + +**Reading the rings.** The ring points come from one of two measurements, both accumulated over **every processed image** rather than one. The default reads the **azimuthally-binned profile (§2) summed over the run**: for each ring and each azimuthal sector, the radial peak is fitted against a locally interpolated background and the measured $(q,\phi)$ mapped back through the current geometry to the pixel it came from. The alternative pools the **spot lists**, which samples each arc wherever the spot finder's threshold happens to bite. The accumulated profile is the same size however many images went into it; the pooled spot list is capped, each image contributing an equal share. + +A plain radial profile — one azimuthal sector — has averaged the ring over every direction and carries no centre at all, so the profile route requires at least four sectors and uses 32 by default. Sixteen to thirty-two are enough; beyond that the limit is the ring's own texture, not counting statistics. + +The extraction window around a ring is capped at half the gap to its neighbour, because the background under a peak is taken from the ends of that window: hexagonal ice has three rings within 0.06 Å⁻¹ of one another, which a fixed window merges into a single peak. Where only one ring is in reach the two tilts are held at their input values rather than fitted, since on a single ring they are degenerate with the centre (above) and the fit would otherwise trade the centre away for them. + +--- + +## 8. Reflection prediction + +Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model. + +### 8.1 Enumerating reciprocal lattice points + +For a maximum resolution $d_\mathrm{min}$, accept $(h,k,l)$ such that: +$ +\lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. +$ + +### 8.2 Still prediction (excitation-error cutoff) + +For still images, the diffracting condition is approximated by an excitation-error cutoff: +$ +\left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. +$ +Accepted reflections are projected to the detector by intersecting the diffracted direction $\mathbf{S}=\mathbf{S}_0+\mathbf{p}$ with the detector plane, using the current geometry. + +When the beam has a finite energy bandwidth, this window is **broadened radially per reflection**: the cutoff is combined in quadrature with a bandwidth smear, $\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}$, where $\sigma_\mathrm{bw}\propto|p_z|$ (the reciprocal-space depth along the beam, growing as $\sim 1/d^2$). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same $\sigma_\mathrm{bw}$ is deconvolved from the measured profile radius (§11.1), so it is not double-counted. + +### 8.3 Rotation prediction (Laue equation + partiality model) + +For rotation/oscillation datasets, Jungfraujoch solves for rotation angles $\phi$ where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define: + +- rotation axis unit vector $\mathbf{m}_2$, +- $\mathbf{S}_0$ incident vector, +- $\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)$. + +A key quantity is: +$ +\zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad +\mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, +$ +which also appears in XDS as the Lorentz component linked to the rotation axis. + +A Gaussian mosaicity model yields a partiality fraction over an oscillation width $\Delta\phi$: + +$ P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], $ + +with mosaicity $\sigma_M$ in radians. + +Reflections are predicted if they meet minimum $\zeta$ and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area. + +### 8.4 Systematic absences (centering) + +Systematic absences are applied at the centering level (prior to full space-group symmetry) **when the space group is supplied by the user**. With no user-fixed space group, prediction runs in $P$ regardless of the centering the lattice search inferred: the centering-absent reflections are integrated so that the space-group search (§13) can confirm or disprove the centering from the measured intensities, and so that a missed superstructure shows up. For centering symbol $C$: + +- $I$: absent if $h+k+l$ odd, +- $A$: absent if $k+l$ odd, +- $B$: absent if $h+l$ odd, +- $C$: absent if $h+k$ odd, +- $F$: absent if any of $h+k, h+l, k+l$ is odd, +- $R$: absent if $(-h+k+l)\bmod 3 \ne 0$, +- $P$: no centering absences. + +--- + +## 9. 2D Bragg integration (profile fitting over a three-ring ROI) + +Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The **default** extraction is **profile fitting** (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection $(I,\sigma,\text{partiality},d)$, so scaling, the rotation combine (§10.6) and merging consume either unchanged. + +### 9.1 Regions of interest + +For each predicted reflection at $(x_p,y_p)$, define three radii: + +- $r_1$: inner signal radius, +- $r_2$: inner background radius, +- $r_3$: outer background radius. + +Pixels are classified by their squared distance $r^2=(x-x_p)^2+(y-y_p)^2$: + +- **signal region:** $r^2 < r_1^2$, +- **background annulus:** $r_2^2 \le r^2 < r_3^2$. + +Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk ($r`, default 0).** The three radii above are fixed pixel counts, identical for every reflection at every resolution. A reflection is not round, though: a finite bandwidth streaks it radially by $\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}$. On a radially smeared spot the fixed $6\ldots13$ px ring therefore sits only $\approx1.3$–$2.2$ radial $\sigma$ from the centre — on the reflection's own tails, which it then measures as background. + +With $k>0$ the **background ring becomes an ellipse**, elongated along the beam→reflection direction by $k\sigma_\mathrm{bw}$. The **radial** semi-axes become $r_2+k\sigma_\mathrm{bw}$ and $r_3+k\sigma_\mathrm{bw}$; the **tangential** half-widths stay $r_2$ and $r_3$; and the growth is capped at $2r_3$, which bounds what a mis-declared bandwidth can do to the bounding box. Pixels are then classified as + +- **signal region:** $r^2 < r_1^2$ — a circle, unchanged, +- **background ring:** $r^2-q_\mathrm{in}\rho^2 \ge r_2^2$ **and** $r^2-q_\mathrm{out}\rho^2 < r_3^2$, + +where $\rho$ is the pixel's radial offset (its projection on the beam→reflection direction), $g=\min(k\sigma_\mathrm{bw},\,2r_3)$ is the capped growth, and $q=1-\big(r/(r+g)\big)^2$ for the boundary concerned. Written this way $k=0$ gives $q=0$ and both tests collapse onto $r^2$ **exactly in floating point**, so the default classifies every pixel exactly as the circular stencil did. The neighbour exclusion above follows: each neighbour's **inner ellipse**, taken in that neighbour's own radial frame, is what is masked out of this reflection's ring. + +The width is the bandwidth streak alone, and deliberately **not** the profile's full radial variance of §9.3, which also carries the sensor parallax and weak-spot capture terms. Those two are the only terms there are on a monochromatic beam, and widening the ring by them was measured on the rotation battery: it neither helped the crystals with clean high-resolution shells nor left the weak ones alone. The bandwidth streak, by contrast, is a measured elongation of the recorded spot — principal axis along the radius to within a couple of degrees, and azimuth-independent. Keeping only it also makes the option exactly inert on a monochromatic beam, where $\sigma_\mathrm{bw}$ is zero. + +Growing the ring also grows the neighbour exclusion, so on a crowded pattern fewer background pixels survive; a reflection left with too few is rejected outright. On the data this was measured on the loss is under 0.1% of reflections, but it is not structurally zero. + +Only the ring moves. The signal disk $r_1$ stays circular, deliberately: it sets $n_S$ and it sets $\mathrm{var}(\hat b)$, and with `--integrator boxsum` it drives the all-or-nothing “every signal pixel valid” acceptance gate (§9.2), so growing it would reject any box sum carrying a single bad pixel anywhere along a long streak. What a circular $r_1$ loses is flux, and that loss is a function of resolution alone, which the per-shell scale absorbs. + +### 9.2 Box summation (seed and fallback) + +Let: +- $S = \sum I(x,y)$ over signal pixels, +- $n_S$ = number of valid signal pixels, +- $B = \sum I(x,y)$ over background pixels, +- $n_B$ = number of valid background pixels. + +Background per pixel and integrated intensity: +$ +\hat{b} = \frac{B}{n_B},\qquad +\hat{I} = S - n_S \hat{b}, +$ +with a Poisson-like uncertainty $\sigma(\hat{I})=\max\!\big(1,\ r_\sigma\hat{I},\ \sqrt{S + n_S^2\,\mathrm{var}(\hat{b})}\big)$, i.e. $\sqrt{S}$ floored both at 1 and at a small fraction $r_\sigma$ of the intensity. The second term under the root is the **uncertainty of the background estimate itself**: $\hat b$ is measured from a finite number of ring pixels, $\mathrm{var}(\hat b)=\hat b/n_B$, and it is subtracted $n_S$ times over, so it enters squared. Omitting it understates $\sigma$ by $\sqrt{1+n_S/n_B}$ — 1.109 with the shipped circular stencil; with an elongated ring $n_B$ grows with resolution, so the factor is no longer one number for a run — uniformly, on every reflection of every dataset. The same term is carried into the profile fit (§9.3), where it adds $(\sum wP/\sum P^2/v)^2\,\mathrm{var}(\hat b)$; $n_B$ is the count of pixels behind the *final* background value, so a clip or trim that discards ring pixels raises it. A box sum is accepted as “observed” only if all signal pixels were valid and $n_B$ exceeds a minimum — it measures what is in the disk with no model of what should be there, so it cannot renormalise a disk it has lost pixels out of. The profile modes can, and do (§9.3). This box sum is the classical estimator; it is used directly with `--integrator boxsum`, and otherwise seeds the profile fit below, where $S$ and $n_S$ then count only the pixels that were actually read. + +**High-side clipped background (default on).** Because $\hat{I}=S-n_S\hat{b}$ is a small difference of large numbers for weak reflections, a per-pixel background bias $\delta\hat{b}$ becomes a *fractional* intensity bias $\approx n_S\,\delta\hat{b}/\hat{I}$ that grows as $\hat{I}$ shrinks — worst at the resolution edge. A plain ring mean reads high there, because neighbour-spot wings that survive the signal-disk mask, tails and zingers are one-sided (positive) contaminants. The ring mean is therefore made robust: pixels above $\hat{b}+n\sqrt{\hat{b}}$ are rejected and the mean recomputed, with $n=4$ (`--background-clip`; $n=0$ disables), lowered by `rugnux` to $n=3$ on broadband (non-zero bandwidth: pink-beam / DMM) data, where a bandwidth-streaked high-resolution spot leaks into the ring more readily. That is only a default — the flag sets $n$ whatever the bandwidth is. A clean Poisson ring is essentially unchanged by the cut (measured false-rejection rate 0.04–0.39 % at $4\sigma$), while a 40-pixel neighbour core at $+100$ counts shifts the estimate by $+0.009$ ct/px. + +The clip cuts only the high tail, which matters: the **symmetric** trimmed mean it replaced (drop the lowest and highest fraction $f$ of ring pixels, $f=0.10$; still reachable with `--background-trim`, which switches the clip off) is *not* a consistent estimator of the mean of a right-skewed Poisson sample. It sits $\approx0.1$ ct/px **below** the true mean at every level, and with $n_S\approx50$ signal pixels in the $r_1$ disk that under-estimate adds $\approx5$ counts to **every** partial — negligible at low resolution, but a large fraction of a partial in the outermost shell. The trim also collapses once contamination exceeds $\approx10\,\%$ of the ring, where the clip does not. Note that removing a positive background bias *lowers* $\langle I/\sigma\rangle$ and *raises* edge $R_\text{meas}$, because both are inflated by information-free counts — so neither may be read as evidence against the change. + +Both estimators are computed in the shared background pass, but only the trim reaches plain box summation: the high-side clip is skipped for `--integrator boxsum`, which therefore uses the plain ring mean unless `--background-trim` is given. + +**Radial background correction (opt-in).** A ring mean estimates the background *under* the signal disk correctly only if the background is flat there. The signal disk and the ring are concentric, so for a background that is **linear** in position $\langle B\rangle_\mathrm{ring}=\langle B\rangle_\mathrm{disk}$ identically — a plane or gradient fit buys exactly nothing. The leading error is the **curvature** of the radial background, which is negligible on a smooth background but reaches tens of counts on a single reflection sitting on a sharp powder ring. That error is a kernel over radial offset, +$ +\delta \hat b \;=\; \textstyle\sum_k \kappa_k\, \bar B(r_0+k), +$ +with $\kappa$ the annulus-minus-disk histogram of the stencil over radial offset, averaged over azimuth, and $\bar B(r)$ the image's own radial background curve. With the fixed circular stencil ($k=0$, §9.1) that single kernel serves every reflection. An elongated ring does not: its radial-offset histogram depends on how far that particular reflection's ring was grown, so $\kappa$ becomes a small table of kernels, indexed by the growth rounded to whole pixels. The azimuthal average survives the change unaltered, because the stencil is rebuilt in the reflection's own frame at each azimuth and so stays radially aligned: what is averaged over is the sub-pixel phase of the detector grid against the radius, which is what genuinely differs between reflections. Applying it costs one short dot product per reflection and no extra pixel reads; correcting the background *scalar* means the box sum, the profile fit and the variance all pick it up. The curve is accumulated from the same annulus pixels the background pass already reads (a pixel's radius is the reflection's radius plus the pixel's projection on the beam→reflection direction, so no per-pixel square root is needed) and specifically from the **clipped** pixels, or it would carry neighbour tails and zingers — which is why the correction is inert under `--integrator boxsum`, that path having no clip pass. + +The model is a function of **radius alone**, so it is applied only where that is true of the background. `--background-radial` takes `on`, `off` or `auto`. It is **off by default**; under `auto` each image's peak-excluded ice score (§3.3) is taken after spot detection and before integration, and the correction is applied to that image when the score reaches the same `--ice-min-score` gate. Smooth powder ice *is* a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes **43 % of the bands' excess amplitude**, and the improvement is **7× larger inside the bands than outside**, which is its stated mechanism; on a crystal whose ice is textured the same correction *increased* the excess amplitude by half; on a clean crystal it is inert to four decimal places. Auto engages only where a peak-excluded score exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off. + +### 9.3 Profile-fitted extraction (default) + +A fixed signal disk captures a *width-dependent* fraction of each spot, which puts a multiplicative floor on the per-observation precision of strong reflections and weights weak reflections poorly. Profile fitting removes this by extracting each intensity against a fitted spot shape, without needing reference intensities. Per frame: + +1. **Seed.** Box-sum every reflection (§9.2) to get a rough intensity and observed centroid, and select strong spots (significance $\ge 5$). +2. **Build the profile.** For `gaussian` (the default) the width is taken **per resolution shell** from the measured second moments of the strong spots (shell-dependent because spot size grows with resolution). The moments are **anisotropic**: each strong spot's pixels are rotated into its *own* radial/tangential frame before being accumulated, giving $\sigma^2_r$ and $\sigma^2_t$ separately. Stacking the spots in the detector frame instead — they sit at every azimuth — averages the two directions away, leaving only $\sigma_r^2+\sigma_t^2$, so radial smearing is read back as a wider *tangential* spot. For `empirical` the profile is instead the averaged, background-subtracted pixel grid of the shell's strong spots, accumulated in the detector frame on their **rounded predicted** positions. For `gaussian` only, the profile is then **rebuilt for each reflection**, centred on its **sub-pixel predicted position** (the noise-free geometric centre, not the observed centroid) and, where needed, **elongated only along the radial direction** (away from the beam centre) — because two effects stretch a spot radially but not tangentially: + - a finite energy **bandwidth** smears each spot by $\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}$ ($R_\mathrm{px}$ = distance from the beam centre, large at high resolution), and + - sensor **parallax** — the depth over which a photon converts in a thick Si/CdTe sensor — adds a term $\propto\tan^2(2\theta)$ (material- and energy-dependent), plus a small fixed weak-spot capture term. + + The two enter as a floor on the measured radial excess: $\sigma^2_\mathrm{radial}=\sigma^2_t+\max\!\left(\sigma^2_r-\sigma^2_t,\ \sigma_\mathrm{bw}^2+c_\mathrm{par}\tan^2(2\theta)\right)$, tangential unchanged at $\sigma^2_t$. The measured excess is what the signal disk can resolve; the analytic term takes over for a streak too long to be measured there. The Gaussian is built on a grid grown to hold the streak — capturing it without the tangential background an isotropic widening would add. The `empirical` profile keeps the fixed per-shell grid and gets none of this. +3. **Fit (Kabsch).** With profile $P$, background $B$ and the shell variance model, the intensity and its uncertainty are +$ +I = \frac{\sum P\,(c-B)/v}{\sum P^2/v},\qquad +\sigma = \sqrt{\frac{1}{\sum P^2/v}},\qquad +v = \max\!\left(B + I\,P,\ \tfrac{1}{2}B\right), +$ +where $c$ is the pixel value and the de-biased variance $v$ (background plus model signal, rather than the down-fluctuating observed count) is iterated (a few passes). The plug-in $I$ enters **as it is**: half-wave rectifying it, $v=B+\max(I,0)P$, lets $v$ — and with it the reported $1/\sum P^2/v$ — respond only to *upward* fluctuations of a noisy estimate, which adds $\approx0.4\,\sigma\sum P^3/(\sum P^2)^2$ to every $\sigma$ whatever the count rate. That offset is invisible on strong reflections and a large fractional inflation on weak ones; the $\tfrac12 B$ clamp keeps $v$ positive without reintroducing it. As a guard, if the profile intensity runs away from the box-sum seed (by more than ~10 box-sum $\sigma$) it falls back to the seed, and the background term is floored at $0.01$ ct/px — enough to keep $P^2/v$ finite when the ring mean reads exactly zero, which a ring of $n_B$ pixels cannot distinguish from any background below $\approx1/n_B$. The rotation/excitation partiality is carried exactly as in the box-sum path. + +**Pixels the fit cannot use (MINPK).** A profile fit is the amplitude of a *normalised* profile, so a pixel left out of the sum renormalises the estimator by construction: it costs information — $\sum P^2/v$ shrinks and $\sigma$ grows — but biases nothing. That is what keeps a reflection whose signal disk is cut by a mask, an untrusted region, a detector gap or an overload: those pixels are simply not read, and the fit is taken over the rest, exactly as the shared pixels of a crowded reflection are (`--overlap exclude`). The reflection is kept only while enough of the expected profile survives — at least `--overlap-minpk` of the profile mass that falls on the detector at all, default 0.75, which is XDS's `MINPK` and dials' `valid_foreground_threshold`. The complete reflections alone teach the profile, its resolution shells and their widths. `--integrator boxsum` has no profile to renormalise with and keeps the all-or-nothing rule of §9.2. + +"Biases nothing" holds only while the profile *model* is exact. Lose the peak and the amplitude is set by the wings alone, so the result stops being a measurement of the reflection and becomes a measurement of how well the fitted shape describes it. The worst case is a pixel invalidated *by the flux it saw* — a detector's per-frame overload marker: that pixel goes missing **because** the reflection was bright, so the loss concentrates on the strong low-resolution reflections that are the largest terms of $R_\mathrm{meas}$, where the fit reads $-50\%$ against the symmetry mates. MINPK cannot catch it, because it cuts on profile *mass* and the peak of a broad spot is a few percent of the mass. So a second condition applies alongside it, on any unreadable pixel whatever made it unreadable: **no unreadable pixel may carry more than 0.9 of the profile's own peak value**. As a fraction of the peak rather than a radius in pixels, that scales with the spot — for a Gaussian it is a cut at $\sqrt{-2\ln f}\,\sigma = 0.46\sigma$, the peak pixel alone where $\sigma$ is 0.8 px and the crest of the ridge where the profile is a bandwidth streak — and it costs well under 0.1 % of the recovered observations. + +The integrator is selected by `--integrator boxsum|gaussian|empirical` (default `gaussian`). + +### 9.4 Lorentz–polarization factor handling + +For integrated reflections, polarization correction can be applied as a multiplicative correction to the reflection scale via the geometry-based polarization term (§2.2). A Lorentz-like factor is carried as `rlp` in predictions, and used during scaling/merging (§10). + +--- + +## 10. Scaling and merging + +After per-image integration, Jungfraujoch scales observations and merges them into unique reflections. The design is intentionally compatible with XDS/XSCALE concepts, and handles both still and rotation data. + +### 10.1 Observation model + +For an observation $j$ of a unique reflection $h$ on image (or image group) $i$, the predicted measured intensity is modeled as: +$ +I_{ij} \approx G_i \, L_{ij}\, P_{ij}\, I_h, +$ +where: + +- $G_i$ is the image scale factor, +- $L_{ij}$ is a Lorentz-like / geometry factor; predictions carry its **reciprocal** as `rlp`, so $L = 1/\texttt{rlp}$ and the correction below is applied as a multiplication by `rlp`, +- $P_{ij}$ is a partiality term (model-dependent), +- $I_h$ is the merged (true) intensity parameter for that unique reflection. + +A least-squares objective is minimized: +$ +\sum_{ij} \left(\frac{I_{ij}^{\mathrm{pred}} - I_{ij}^{\mathrm{obs}}}{\sigma_{ij}}\right)^2 +$ +solved by robust (Cauchy) weighted least squares, with optional post-fit smoothing of the per-frame scales for rotation series (§10.3). + +### 10.2 Partiality models + +The partiality applied is fixed by the data type and scaling stage, not chosen from a user menu: + +1. **Rotation partiality** (XDS-like; see §8.3), used for the per-frame scaling of rotation partials: + $ + P_{ij} = \frac{1}{2}\left[ + \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - + \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) + \right]. + $ + The mosaicity $\sigma_{M,i}$ is **measured once per image at indexing** (MLE, §11.2) and held fixed during scaling — only smoothed in frame order (§10.3), never re-refined (it is degenerate with the scale $G$; §11.2). + +2. **Unity** ($P_{ij}=1$): used for the scale-on-fulls refit (§10.6), where each observation is already a complete reflection. + +3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius (resolution-independent), the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature. The fit typically drives $\gamma_e\to0$, leaving the resolution-independent $\gamma_0$ as the effective width. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times. + +Reflections below a minimum partiality can be rejected from merging to avoid unstable corrections. + +### 10.3 Smoothing of per-frame scales + +The per-frame scales $G_i$ are fit by robust (Cauchy) inverse-variance-weighted ratios; there is no explicit $G\approx1$ prior. For rotation datasets, optional smoothing enforces the expectation that scale and mosaicity vary slowly across a sweep: **after** the per-frame fit, $\log G_i$ (and the mosaicity) are replaced by a centred **moving average** over a window spanning a configurable rotation range (XDS DELPHI-like; `--smooth-g`, default 5° for rot3d, off otherwise). It is a post-fit smoothing pass, not a curvature penalty inside the least-squares objective. + +The **crystal orientation** is smoothed the same way, and for the same reason. Geometry is re-refined independently on every frame against that frame's spots alone — as few as a dozen on a sparse crystal — so the per-frame orientation carries a real slow drift (crystal slippage, up to ~1.3° across a sweep) on top of fit noise that scales with spots per frame. Before scaling, the per-frame lattices are de-rotated to a common reference, averaged in frame order, rotated back, and every partial's $\Delta\phi$ — hence its partiality — is recomputed from the smoothed lattice. The window is chosen per dataset by leave-one-out cross-validation (does a frame's neighbours predict its orientation?) rather than fixed, because drift and noise both vary by two orders of magnitude between crystals; it is capped, because the per-frame fit also absorbs a real per-frame systematic that smoothing too wide destroys. Only frames that actually indexed take part: a frame that did not carries an all-zero lattice, which is *finite* and so passes a validity check written as a finite test, and would otherwise be both averaged into its neighbours' orientation and scored in the cross-validation that picks the window. Refining *less* is not an alternative: with per-image refinement off the space group is lost on several crystals. + +A per-frame scale enters every intensity as $1/G$, so a frame whose fit is not determined by its data can amplify it without bound — and $\sigma$ is amplified by the same factor, which makes it invisible to any $\sigma$-based outlier test. A fitted $G$ far below the run's median is therefore treated as *undetermined* rather than as a successful fit, both here and in the separate refit on the combined fulls (§10.6). The bound is a ratio to the run's own median because $G$ is not gauge-fixed: it and the merged means have an exact global multiplicative degeneracy, so no absolute value is meaningful. + +### 10.4 Merging estimator + +After refinement, corrected observations are formed: +$ +I^{\mathrm{corr}}_{ij} = \frac{I^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}},\qquad +\sigma^{\mathrm{corr}}_{ij} = \frac{\sigma^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}}. +$ + +Unique intensities are merged by inverse-variance weighted mean: +$ +I_h = \frac{\sum_j w_j I^{\mathrm{corr}}_{ij}}{\sum_j w_j},\qquad +w_j = \frac{1}{(\sigma^{\mathrm{corr}}_{ij})^2}. +$ + +The weights use an **expected** variance: the Poisson signal part of each $\sigma^{\mathrm{corr}}_{ij}$ is rebuilt at the reflection's merged $\langle I\rangle$ rather than at that observation's own intensity. Weighting by an observation's own $\sigma^2$ biases the inverse-variance mean low below about one photon, because an up-fluctuated observation gets a larger sigma and is then down-weighted too hard. The rotation combine already does this; for stills it is on by default, and `--no-expected-variance-merge` restores the observed-sigma weighting. + +An internal-consistency term can inflate uncertainties when multiple observations are present, in the spirit of XSCALE. + +### 10.5 Merging statistics + +Per-shell and overall merging statistics are computed on corrected intensities, including: +- number of observations and of unique reflections, and multiplicity, +- mean $I/\sigma(I)$, +- $R_\mathrm{meas}$ (the redundancy-independent Diederichs–Karplus form) from within‑HKL deviations, +- $\mathrm{CC}_{1/2}$ (half-set correlation) and, when a reference dataset is supplied, $\mathrm{CC}_\mathrm{ref}$, +- completeness against the enumerated reflections for the cell and symmetry, +- the anomalous signal-to-noise $\mathrm{SigAno}$ (below). + +The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly. + +$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. + +**Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ("$\mathrm{CC}_\mathrm{anom}$") is **not** reported. Its two half estimates $\Delta I_0,\Delta I_1$ are complementary partitions of one observation pool ($\Delta I_0+\Delta I_1=2\,\Delta I_\mathrm{full}$), and subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, leaving the small anomalous signal against the per-half split noise; below an anomalous signal-to-noise of $1$ per half that correlation tends towards $-1$ rather than $0$. $\mathrm{SigAno}$ has no such floor. It is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the printed merge-statistics table. + +### 10.6 Rotation datasets: combining partials into fulls (3D integration) + +In a rotation scan a reflection is recorded as a series of *partials* spread across the frames its rocking curve crosses. Merging those partials directly would force the merge error model to absorb the rocking-curve slicing as if it were measurement noise, capping the achievable $I/\sigma$. For rotation data Jungfraujoch instead **combines** each reflection's partials into a single *full* intensity first, then scales and merges the fulls — a 3D integration over the rocking curve. + +The combine groups each reflection's partials into rocking events (contiguous runs of frames) and reduces each event to one full: + +- **De-biased weighted sum.** Partials are combined by inverse-variance weighting, where each partial's variance is its background-noise component plus the *model* signal shared across the event (Kabsch profile-fit form). Using the shared model signal rather than the individual down-fluctuating intensity stops weak partials from being over-weighted, which would otherwise inflate the merged error model. The weights depend on the full, so the estimate is iterated. +- **Captured fraction.** The partiality summed over the event, $f=\min(1,\sum_j p_j)$, measures how completely the rocking curve was sampled. A full whose curve was captured below a threshold (`--min-captured-fraction`, default 0.7 for rotation) is dropped — an event seen over only a small fraction of its curve is unreliable however many frames it spans. (The per-partial minimum-partiality cut of §10.2 still applies upstream, in the per-frame scaling.) +- **Per-image rejection (opt-in).** A frame whose observations correlate poorly with the merged reference is not measuring the crystal being merged — it may be off-crystal, or on a *different* crystal where two lattices occupy separate regions of the sample. `--min-image-cc` drops such frames. It has no default: the per-frame correlation measures data quality as much as frame validity, and its typical level varies widely between datasets, so no single absolute bound is generally valid. +- **Capture-aware uncertainty.** A full captured incompletely ($f<1$) is extrapolated and biased high. The unobserved fraction is charged as an extra systematic uncertainty, $\sigma^2 \leftarrow \sigma^2 + \big(c\,(1-f)\,I\big)^2$, so the merge down-weights these extrapolated fulls and the error model treats their scatter as expected. It is enabled by default for the rotation path. + +The fulls are then re-scaled in the XDS sense — a per-image scale refit directly on the complete reflections under the unity partiality model — and merged (§10.4). Because every merged observation is now a counting-statistics-limited full rather than a partiality-divided slice, the error model reaches a far higher asymptotic $I/\sigma$. + +After scale-fulls, four **correction surfaces** are fitted on the combined fulls (rotation path, **on by default**; disable all with `--no-scaling-corrections`), each an alternating multiplicative refinement of the per-full scale against the merged reference: + +- **Decay.** Radiation damage weakens later frames more at higher resolution — a resolution×time (Debye–Waller) systematic the resolution-flat per-image scale cannot capture. A single global relative-$B$ rate is fitted, $\ln(I_\mathrm{ref}/I_\mathrm{obs}) = 2\,(\mathrm{d}B/\mathrm{d}n)\,(n-\bar n)\,s^2$ (frame $n$, $s^2 = 1/4d^2$), and folded into the scale. It engages only when the total relative-$B$ over the run exceeds a physical floor (2 Ų); below that the decay is negligible and "correcting" it only spreads symmetry equivalents (same $s^2$, different frames). An optional **per-batch relative-$B$** (`--relative-b[=deg]`, off unless requested; 10°-of-rotation batches by default) extends the single global rate to a smooth $B(n)$ curve — the same $s^2$-weighted decay fit solved independently over short frame batches, curvature-penalized so it cannot over-fit and cross-validated like the surfaces below — for crystals whose decay is non-linear in dose. Its cross-validation splits on **ASU-group parity**, not the frame parity the surfaces below use: a per-batch parameter owns whole frames and so cannot be scored on a held-out frame, whereas splitting the symmetry equivalents tests whether a batch's $B$ generalises to reflections it was not fitted on. +- **Absorption.** A smooth multiplicative factor over the diffracted-beam direction expressed in the goniometer (crystal) frame: each full's predicted detector position gives the lab diffracted direction, de-rotated by the spindle so a fixed crystal-frame direction is sampled at many rotation angles and its grid cell is well-determined. Negligible at hard X-rays / thin crystals; it matters at low photon energy. +- **Modulation** (detector-plane flat-field). A smooth multiplicative factor over where each reflection lands on the detector (predicted $x,y$): symmetry-equivalents land at different positions as the crystal rotates, over-determining the surface. It absorbs detector-response and geometric systematics that inflate $R_\mathrm{meas}$. +- **Time-dependent absorption.** The same surface as *Absorption*, but over (rotation angle × detector position) instead of the crystal-frame direction alone. The two agree while the illuminated volume stays put — the incident path then depends only on the spindle angle, which the per-image scale already takes, and the exit path is fixed in the crystal frame. Once the diffracting volume drifts through the beam the exit path becomes a function of the spindle angle as well, and nothing time-independent describes it. Fitted last, on 12 rotation bins × a 10×10 equal-occupancy detector grid, so the two time-independent surfaces get first claim on what they can explain. + +Each surface is **cross-validated**: fitted on even-numbered frames and kept only if it improves the held-out odd-frame agreement by a clear margin (and vice versa), scored by a **σ-independent, $R_\mathrm{meas}$-like** fractional agreement $\sum|I_s-I_\mathrm{ref}|/\sum|I_\mathrm{ref}|$ rather than a studentized $\chi^2$ — so a surface cannot "pass" by reshaping the sigmas instead of tightening the intensities. A surface fitted to noise where its systematic is absent does not generalize and is discarded — a correction never adds scatter. + +**Radiation-damage report (rotation, report-only).** Independently of whether any decay correction is applied, rugnux measures and reports the relative Debye–Waller $B$ across the sweep: the per-image scale's correlation to the merge and the per-image mosaicity versus frame (dose), together with a per-batch relative-$B$ curve whose first→last change is a single headline number (measured before any decay correction, against the least-damaged early wedge). It is written to the log and to the merged mmCIF as a data-quality-vs-dose diagnostic and **never** alters the merged intensities — distinct from the decay correction above, which does fold into the scale. + +Each batch's $B$ is fitted on **resolution-shell means**, not on single observations: $\ln(I_\mathrm{ref}/I_\mathrm{obs})$ of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs $I_\mathrm{obs} > 0$, which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely *dimmer* than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as **absent** rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report (`docs/RUGNUX.md`) to name. + +### 10.7 R-free test-set flags + +A fraction of the unique reflections (`rfree_fraction`, default 0.05) is flagged as a **free (test) set**, written to the output (MTZ `FreeR_flag`, mmCIF `_refln.status_free`, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection's **Friedel-merged (Laue) ASU index**, which gives three properties: + +- all symmetry- and Friedel-equivalent reflections share one flag — in particular a Bijvoet pair $I(+)/I(-)$, kept as two separate merged rows in anomalous mode, is **never split** across the work and free sets (which would bias R-free); +- the free/work decision is a deterministic hash of that key, so the same reflection always lands in the same set — reproducible run-to-run and independent of the order in which observations were merged; +- the hash depends only on the reflection index, **not** on this dataset's resolution range or which reflections it happens to contain, so a uniform draw takes ~`rfree_fraction` of the distinct reflections free and — crucially — **every dataset of one crystal form gets the same free set**. That cross-dataset consistency is what a multi-dataset campaign (ensemble refinement, PanDDA) requires; a per-shell stratification tied to each dataset's own $d_\mathrm{min}$ would break it. + +On small data, where `rfree_fraction` (default 0.05) would give too few test reflections for a statistically stable R-free (Brünger's ~500–2000 rule), the fraction is **floored** so at least ~500 distinct reflections are free — capped at 10 % so a large test set never steals working data. For ordinary data this floor is inactive and the fraction stays flat at `rfree_fraction`, preserving the cross-dataset-identical property above; it only lifts the fraction on genuinely small datasets, where per-dataset R-free stability outweighs cross-dataset identity (and a shared reference set is the way to keep exact identity there). + +When a reference MTZ (`--reference-mtz`) carries a `FreeR_flag` column, its test set is **imported** instead: every merged reflection whose Laue-ASU index matches the reference takes the reference's flag (reflections absent from the reference keep the hash flag). This lets a whole fragment-screening campaign inherit one shared free set from the apo/reference dataset. The CCP4/refmac convention (test set = flag 0, including the historical 0–19 form) is assumed, with the complement taken automatically if flag 0 would be the majority (a phenix-style file where 1 marks free). + +### 10.8 French–Wilson amplitudes + +The last step of the merge estimates a Bayesian structure-factor amplitude $|F|$ for each unique reflection from its intensity $I$ and error $\sigma$, so the output carries amplitudes alongside intensities (a naïve $\sqrt{\max(I,0)}$ turns every weak or negative measurement into a biased — or zero — amplitude). With the Wilson prior for the true intensity $J\ge 0$ at that resolution, + +$ +P_\mathrm{acentric}(J) \propto e^{-J/\Sigma},\qquad +P_\mathrm{centric}(J) \propto J^{-1/2}\,e^{-J/2\Sigma}, +$ + +and a Gaussian likelihood $\mathcal{N}(I;J,\sigma^2)$, the posterior mean amplitude and its uncertainty are + +$ +\langle |F|\rangle = \frac{\int_0^\infty \sqrt{J}\,\mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J}{\int_0^\infty \mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J},\qquad +\sigma_F = \sqrt{\langle J\rangle - \langle|F|\rangle^2}. +$ + +The prior mean is $\Sigma = \varepsilon\,\langle I/\varepsilon\rangle_\mathrm{shell}$, where $\varepsilon$ is the reflection's epsilon (symmetry-enhancement) multiplicity and $\langle I/\varepsilon\rangle$ is the Wilson mean in its resolution shell (so reflections on symmetry elements, and each shell, are treated correctly). Strong reflections ($I>4\sigma$) short-circuit to $|F|=\sqrt{I}$, where the French–Wilson bias is negligible; a reflection with an unusable $I/\sigma$ falls back to $\sqrt{\max(I,0)}$. The integral is evaluated numerically with a log-shift for stability. + +Amplitudes are written as MTZ `F`/`SIGF`, mmCIF `_refln.F_meas_au`/`F_meas_sigma_au`, and appended to the text HKL, alongside the intensity columns. The **same** $|F|$ feed the model-validation step (§14), so the reflection file and the maps use one consistent set of amplitudes. + +### 10.9 Reference data: fixing the space group and resolving the indexing ambiguity + +A reference dataset (`--reference-mtz`) supplies known intensities for the same crystal form, and is used in two ways. + +**Fix the space group and cell.** Unless overridden on the command line (`-S` for the space group, `-C` for the cell), the reference's space group is adopted and its cell is used as the soft reference cell — indexing may still drift the cell within tolerance, so a small mismatch between reference and data is absorbed rather than rejected. This applies to both stills and rotation data. + +**Resolve the indexing (merohedral) ambiguity.** When the lattice symmetry is higher than the crystal's Laue symmetry (e.g. $P3$, $P4$, $P6$, $C2$), more than one indexing of the same lattice is geometrically valid, and the two solutions produce *different* merged intensities that a self-consistent scale cannot tell apart — only an external reference can. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation $\mathrm{CC}_\mathrm{ref}$ of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is **metric-preserving** — only the $hkl$ labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined; the reference is *not* used to scale the rotation merge, which stays self-consistent (its $\mathrm{ISa}$ comes from the data alone). For stills the reference is the per-image scale target of the on-the-fly scaling (§10.2). + +### 10.10 Ice rings at the scale and merge stages + +Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q$ of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are **kept in the final merge**, which is also what the established scaling programs do by default, so the affected shells keep their completeness. + +Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge's own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged $\mathrm{CC}_{1/2}$ and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell. + +--- + +## 11. Mosaicity and “profile radius” monitoring + +### 11.1 Profile radius (intrinsic excitation-error width) + +The “profile radius” is the intrinsic angular width of a reflection — crystal mosaicity plus beam divergence — estimated from the spread of $\Delta_\mathrm{Ewald}$ over indexed spots, +$ +R \approx \sqrt{\tfrac{1}{N}\sum_i \Delta_{\mathrm{Ewald},i}^2}. +$ +When the beam has a finite energy bandwidth, that bandwidth smears each reflection radially by $\sigma_\mathrm{bw}\approx \mathrm{bandwidth}\cdot\lambda/2d^2$ (largest at high resolution), which also broadens the measured $\Delta_\mathrm{Ewald}$ spread. Since prediction re-applies the bandwidth term per reflection (§8.2), this contribution is deconvolved from the estimate — $R^2 = \langle\Delta_\mathrm{Ewald}^2\rangle - \langle\sigma_\mathrm{bw}^2\rangle$ — so that $R$ is the intrinsic width and bandwidth is not double-counted. Still predictions use an excitation-error cutoff proportional to $R$. + +### 11.2 Mosaicity from rotation data + +For rotation data the mosaicity $\sigma_M$ is estimated by maximum likelihood from the rocking offsets $\tau$ of indexed spots, using the XDS reflection-fraction model $R(\tau;\sigma_M/\zeta)$ (Kabsch 2010): each spot's exact Bragg angle is located near its frame, $\zeta$ (the rotation-axis Lorentz component) is computed, and $\sigma_M$ is chosen to maximize $\sum_i \log R(\tau_i;\sigma_M/\zeta_i)$. + +The $\phi$ search window for the Bragg angle is set **wider than the oscillation**, so that reflections recorded at large rocking offset are included. These tail reflections carry most of the information about the mosaic width; a window limited to the oscillation range would truncate the $\tau$ distribution and bias $\sigma_M$ low. + +The fit uses only the **strongest 250 spots** of an image, whatever the indexing spot budget (`--max-spots`) is. A spot is detected when $I_\mathrm{full}R(\tau)$ clears the finder threshold, so selecting spots by intensity censors on $R(\tau)$: a deeper list holds proportionally more large-$\tau$ partially recorded spots and the fit widens with it. Left uncapped, $\sigma_M$ therefore tracks the spot budget rather than the crystal — and since an over-wide mosaicity mis-states every partiality, the merge degrades sharply with it. + +The estimated mosaicity feeds the rotation prediction (how many frames each reflection spans, §8.3) and the rotation partiality (§10.2). It is **held fixed during scaling**: in the per-image scale fit the mosaicity is degenerate with the scale $G$ (both rescale the predicted intensity), so refining it there is unstable. A correct mosaicity matters because it controls both how much of each rocking curve is captured and the partiality used to form fulls (§10.6); too small a value truncates the captured curve and over-peaks the partiality, degrading the combined fulls. + +--- + +## 12. Auxiliary statistics: ⟨I/σ(I)⟩ and Wilson plot + +### 12.1 Per-shell ⟨I/σ(I)⟩ + +For monitoring integration quality, Jungfraujoch reports mean $\langle I/\sigma(I)\rangle$ in a fixed number of resolution shells. Shelling is performed in $1/d^2$ space (typical of crystallographic practice). + +### 12.2 Wilson plot (B-factor proxy) + +A Wilson-type analysis is computed by binning intensities by resolution and fitting: +$ +\langle I\rangle \propto \exp\!\left(-\frac{B}{2}\frac{1}{d^2}\right), +$ +i.e. +$ +\log \langle I\rangle = \mathrm{const} - \frac{B}{2}\left(\frac{1}{d^2}\right). +$ +A linear regression of $\log\langle I\rangle$ vs $1/d^2$ provides an estimate of $B$, subject to basic quality checks (e.g. $R^2$ threshold). + +A **dataset-wide** Wilson $B$ is also estimated over the merged reflections — restricted to the meaningful resolution range (skipping the low-resolution non-linear region below ~4 Å and shells past the signal limit $\langle I/\sigma\rangle < 1$, so it is insensitive to how far the merged data extend) — and written to the merged mmCIF as `_reflns.B_iso_Wilson_estimate`, the analogue of XDS's Wilson-line $B$. It is diagnostic only and is not fed back into scaling. The **per-image** estimate (used for the live radiation-damage plot) is accepted only when the fit is well-correlated and physically plausible ($0 < B < 200$ Ų); on a bad frame (an indexing glitch, too few reflections) the Wilson line runs wildly steep, so an implausible $B$ is reported as NaN rather than a spurious hundreds-of-Ų value. + +--- + +## 13. Space-group determination and merge-level decisions + +### 13.1 Space-group determination + +When no space group is supplied, a POINTLESS-like search scores Laue-group symmetry (CC of $I(h)$ vs $I(Rh)$ plus merge self-consistency) and detects screw/centering absences from the $P1$-merged intensities. Three tests gate a promotion to higher symmetry, all aimed at the merohedral twin, whose twin law forces non-equivalent reflections together and so mimics symmetry: + +1. **Merge self-consistency** ($\chi^2$ under the candidate group, relative to the confirmed subgroup). On its own this is not sufficient: it is a ratio to an error model that moves with the *amount* of data — the parent's systematic term grows as $\sigma$ shrinks with $1/\sqrt{N}$, while a twin's is already saturated — so its verdict depends on how much data the search saw. +2. **Error-model $b$** (the intensity-proportional systematic). A genuine symmetry step gains multiplicity without inflating $b$; merging a twin law's extra operator inflates it. A $\chi^2$-passing promotion is vetoed when $b$ rises past a bound relative to the confirmed subgroup. +3. **Operator disagreement**, a sigma-free statistic $H=\mathrm{median}\,|I_1-I_2|/(I_1+I_2)$, formed as the ratio of the operators a promotion *adds* to the parent's own, measured on the same reflections. Normalising against the parent divides out the systematic floor that symmetry mates carry on real data, which varies by crystal and by operator; a median is used because a twin perturbs every pair whereas a badly-measured minority perturbs only the tail. Where a candidate has several parents of the same order, it is judged against the worst of them, since a rival subgroup can itself contain the twin laws. + +The operator correlations are taken on reflections above an $I/\sigma$ cut, and that cut is **capped at the merge's own $I/\sigma$ quantile** rather than applied as a fixed number. On a search merge whose ISa is below 3, a fixed cut of 3 selects nothing at all, leaving every operator correlation undefined and collapsing the point group to 1. The cap keeps at least the strongest quarter and is inert — the cut stays exactly 3.0 — on a healthy merge. + +Several space groups may share an absence pattern exactly. Where they do, the search scores them identically and **all of them are named** in the result rather than one being reported as the answer: some are enantiomorph pairs, which merged intensities cannot distinguish in principle, and others differ only by a screw condition that the centering condition already implies, so the screw has no observable signature at all. The representative reported first is the lowest space-group number, which is a convention and not a measurement. + +The Lorentz factor $\zeta$ (§8.3) governs how well a reflection can be measured, so when the spindle lies in a plane of the lattice, an operator permuting the two in-plane axes samples a different mixture of measurement qualities than one that only flips signs. The search is therefore run a second time on a merge of only the well-measured observations (`--search-min-zeta`, rotation default 0.85), both answers are reported, and **where they disagree the merge of all the observations decides**. The filter discards 40–80 % of the observations, which can starve an operator correlation the full merge confirms and can equally leave an operator confirmed that the full merge refuses, so the decision — the point group as well as the absences, which live in the weak reflections the filter removes — rests on the arm with every observation behind it. A tie (same order, different symmetry) is reported with both candidates named, for trying in molecular replacement. + +**Centering** is accepted when the systematically-absent class is weak relative to the present one by *either* of two floor-independent tests: its mean signed $I/\sigma$ well below the present mean, *or* its rate of individually-significant reflections well below the present class's own significant rate. The second test covers weak and low-energy data, where a positive intensity floor (background and profile leakage) lifts the absent class's mean $I/\sigma$ well above zero and, when the present class is itself weak, carries the plain mean ratio past its bound; a false centering fails both tests, its absent class being as strong as the present one. When several centerings pass, they are ranked by their **net** systematic absences (absent minus violating), not the gross absent count, so a super-centering (e.g. $F$ over a true $C$) whose extra, only-half-populated absent class dilutes the strength ratio does not out-rank the correct lower centering. + +### 13.2 Twinning check + +A Padilla–Yeates $L$-test ($\langle|L|\rangle$, $\langle L^2\rangle$) and the second moment $\langle I^2\rangle/\langle I\rangle^2$ (taken per resolution shell with noise-only shells skipped and Wilson outliers rejected, so a single strong reflection in a collapsed-mean shell cannot skew it) are written to the merged mmCIF as a twinning diagnostic. Twinning is only flagged in Laue classes where a merohedral twin law can exist; the holohedral high-symmetry classes ($4/mmm$, $6/mmm$, $m\bar{3}m$, and $\bar{3}m$ on a rhombohedral lattice) are exempt, so a low $\langle|L|\rangle$ there is reported as a statistical artefact rather than twinning. + +### 13.3 Outlier rejection + +Merging applies an optional per-observation median-based $N\sigma$ cut (`--reject-outliers`, default 6σ for `rot3d`, off otherwise). The same $N\sigma$ cut is fed back into the error model: after an initial $a,b$ fit the parameters are re-fit once on the reflections that survive rejection (dropping any whose squared deviation exceeds $N\sigma^2\,[a\,\sigma^2 + (b\,\langle I\rangle)^2]$), so the calibrated errors describe the reflections that actually enter the merge rather than the pre-rejection pool. +### 13.4 Automatic resolution cutoff + +By default the reported/written high-resolution limit is trimmed where $\mathrm{CC}_{1/2}$ falls off: a logistic is fitted to $\mathrm{CC}_{1/2}(s)$, and the limit is set **one reported-shell width past** the point where the fit crosses 0.30 — deliberately "one shell too far", so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. `--scaling-high-resolution` overrides the limit and `--resolution-cutoff off` disables it. + +### 13.5 Practical notes and limitations + +- **Bragg integration is profile-fitted by default** (per-shell Gaussian profile, Kabsch extraction; §9.3), with plain box summation available as a fallback (`--integrator boxsum`). The profiles are built per frame from that frame's strong spots, which suits fast-feedback and serial/streaming use; a profile shared across many frames (as in full offline workflows) is not currently formed. +- **Space-group symmetry** beyond centering absences is not enforced during prediction/integration unless the space group is supplied and used downstream. +- **Resolution masking** is controllable, and so is every stage of ice-ring handling (§3.3, §10.10). None of it runs unless the crystal is measured to have ice, because the fixed bands are a fixed cost in unique reflections whether it does or not. +- **Rotation vs still modes** differ substantially in prediction and scaling: partiality is angle-driven in rotation data, while stills are predicted within an excitation-error window and get their partiality from the default-on per-crystal tilt post-refinement (§10.2) — or unit partiality with `--simple-stills`. +- **Amplitudes and intensities.** The merged output carries both intensities (mmCIF `intensity_meas`, MTZ `IMEAN`/`SIGIMEAN`) and French–Wilson amplitudes (mmCIF `F_meas_au`, MTZ `F`/`SIGF`; §10.8), so a downstream program can refine against either. + +--- + +## 14. Model-based validation: R-free against a model and electron-density maps + +Offline (`rugnux --model model.pdb`) the merged data can be scored against a supplied atomic model and **initial** electron-density maps computed — enough to confirm that a model fits the data and to inspect the density, not a substitute for refinement. **The structure itself is not refined**; the model is only re-fractionalized into the data unit cell (a rigid cell adjustment, so a deposited model with a slightly different cell still lines up), and the observed amplitudes are the French–Wilson $|F|$ from §10.8, so the R-free and the maps use exactly the same amplitudes as the written reflection file. The model, structure-factor, bulk-solvent and FFT machinery is provided by GEMMI. + +### 14.1 Model structure factors + +The model electron density is sampled on a grid (IT92 X-ray form factors, with a Refmac-compatible Gaussian blur chosen for the grid spacing) and Fourier-transformed to structure factors $F_\mathrm{calc}(hkl)$ up to the data resolution. + +### 14.2 Bulk solvent and scaling + +A flat bulk-solvent mask around the model is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$: + +$ +F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2. +$ + +This is the standard, few-parameter scaling model used by refinement programs. No free-form per-resolution-shell rescale is applied: such a rescale is dataset-specific and reshapes each map's radial amplitude profile differently, which would make maps from a multi-dataset campaign no longer directly comparable. + +### 14.3 R-work and R-free + +Crystallographic R-factors are reported over the work and free sets (the §10.7 flags): + +$ +R = \frac{\sum \big|\,|F_o| - |F_\mathrm{model}|\,\big|}{\sum |F_o|}, +$ + +with R-free the same sum restricted to the free set. Note that the scaling of §14.2 is fitted over **all** reflections, work and free alike — its few parameters ($k$, an anisotropic $B$, $k_\mathrm{sol}$, $B_\mathrm{sol}$) are far too few to absorb individual reflections, but R-free here is strictly "free of refinement", not free of the scaling fit. + +### 14.4 Electron-density maps + +Two maps are formed with the model phases $\varphi_\mathrm{model}$: a $2F_o-F_c$ map, coefficients $(2|F_o|-|F_\mathrm{model}|)\,e^{i\varphi_\mathrm{model}}$, and an $F_o-F_c$ difference map, $(|F_o|-|F_\mathrm{model}|)\,e^{i\varphi_\mathrm{model}}$, each inverse-Fourier-transformed to a real-space CCP4 map (`_2fofc.ccp4`, `_fofc.ccp4`). A map-coefficient MTZ (`_maps.mtz`: `FP`, `FC`, `PHIC`, `FWT`/`PHWT`, `DELFWT`/`PHDELWT`, `FREE`) is written alongside so the maps can be reopened or rebuilt in Coot / PyMOL. These are unweighted difference coefficients (no $\sigma_A$ / figure-of-merit weighting), which is why they are described as *initial* maps. + +### 14.5 Aligning the data to the model: enantiomorph and indexing ambiguity + +The model fixes a definite hand and indexing, but the merged data need not share them, so before comparison the observed reflections are brought into the model's frame. + +- **Enantiomorph / screw.** When the data space group is the enantiomorph of the model's (e.g. data $P4_12_12$, model $P4_32_12$; or $P3_1/P3_2$), the two are **indistinguishable from merged intensities** — $|F_\mathrm{calc}|$ is invariant under the change of hand, so R-free cannot choose between them and probing would be meaningless. The hand is therefore taken from the model: the observed reflections are reindexed by the change-of-hand operator into the model's enantiomorph. Only the map phases (the density's hand) depend on this choice. +- **Indexing (merohedral) ambiguity.** When the crystal has a merohedral ambiguity (§10.9), the observed intensities *do* differ between indexings, and the right one is chosen against the best available reference. **If a reference MTZ was supplied, the data were already reindexed to agree with it** (§10.9 — by the reference-intensity correlation, at the merge stage for rotation data or per image in stills scaling), and model validation keeps that authoritative choice. **Only with a model and no reference** does validation resolve the ambiguity itself, as a fallback: the scaled model is fit to each reindexing of the data (identity plus the twin-law cosets) and the one giving the **lowest R-free** is kept. This matters for a multi-dataset campaign — a single shared reference fixes one indexing convention for every dataset, whereas an independent per-dataset lowest-R-free choice could send borderline datasets to different conventions. A no-op either way for a holohedral crystal (no twin laws). diff --git a/_sources/DEPLOYMENT.md.txt b/_sources/DEPLOYMENT.md.txt new file mode 100644 index 00000000..12a394ec --- /dev/null +++ b/_sources/DEPLOYMENT.md.txt @@ -0,0 +1,168 @@ +# Deployment + +To deploy Jungfraujoch, one needs to follow four steps: + +1. Install main Jungfraujoch code and frontend web interface +2. Flash the U55C FPGA card with a proper image and install Linux kernel driver +3. Install Jungfraujoch writer +4. Install Python OpenAPI client + +Installation procedure depend a lot on the operating system. For RedHat Enterprise Linux 8/9, Rocky 8/9, +Ubuntu 22.04/24.04 or compatible, installation can be done with prebuilt packages from the +[package repositories](REPOSITORIES.md) and is relatively straightforward. For other systems one needs +to build software from source. Both ways will be presented. What each released package contains, and +what it needs on the machine, is described in [Release contents](RELEASE_CONTENTS.md). + + +## Install main Jungfraujoch code and frontend web interface + +On RHEL 8 systems there is a `jfjoch--1.el8.x86_64.rpm` that needs to be installed and contains all the necessary software and web interface. + +On other OSes one needs to compile Jungfraujoch from source (from the repo directory): +``` +$ mkdir build +$ cd build +$ cmake .. -DCMAKE_INSTALL_PREFIX= +$ make +$ sudo make install +``` +For manual installation, we recommend to use non-standard directory (like `/opt/jfjoch`), to facilitate upgrades and removal. +For DKMS to manage kernel module sources it is necessary to copy driver sources to `/usr/src/jfjoch-` directory. This requires extra flag in cmake `-DJFJOCH_INSTALL_DRIVER_SOURCE=ON`. + +Frontend web user interface has to be built separately with: +``` +$ cd build +$ make frontend +``` +Frontend files (.html and .js) will be placed in `frontend/dist` (outside of `build/` directory!) and has to be copied to a general location, e.g. `/usr/local/jfjoch/frontend` or `/opt/jfjoch/frotend`. + +## Flash the U55C FPGA card with a proper image and install Linux kernel driver. + +### Firmware flashing +1. Check that the card is detected by OS with "lspci |grep Xilinx" and check the PCIe bus/device/function (BDF) number, `11:00.0` in this case: +``` +$ lspci |grep Xilinx +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2) +``` +Note the device number `3450` that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and `rev 2` identifying release of the firmware. + +2. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given): +``` +$ sudo lspci -vv -s +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 +(...) +LnkSta: Speed 16GT/s (ok), Width x8 (ok) +(...) +``` + +3. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn't allways reach correct timing). +4. Flash the card with `xbflash.qspi` tool (part of Jungfraujoch). For fresh card use: +``` +sudo xbflash.qspi --primary --card --bar-offset 0x1f06000 +``` +For card that was already flashed with Jungfraujoch images: + +``` +sudo xbflash.qspi --primary --card +``` +It is necessary to confirm the operation by pressing `Y` key or one can add `--force` option to avoid confirmation. +It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions. + +5. Cold reboot: +``` +sudo ipmitool chassis power cycle +``` + +### Install PCIe driver + +For first run it is though recommended to try the driver without installing to the kernel directory: +``` +$ cd fpga/pcie_driver +$ make +$ sudo insmod jfjoch.ko +``` + +Check with `dmesg` that the device was properly found: +``` +$ dmesg |grep jfjoch +[ 431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142) +[ 431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a +``` + +If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. +On RHEL 8 you can install the prebuilt `jfjoch-driver-dkms` package from the +[Gitea package registry](REPOSITORIES.md). On other systems follow procedure in +[PCIe driver](FPGA_PCIE_DRIVER.md). + +NOTE: Driver installation procedure on non-RHEL 8 systems is not well understood/optimized at the moment. + +NOTE: In case driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk if driver is updated: +``` +$ sudo dracut -f +``` +### Configure network +Configure switch according to [FPGA network guide](FPGA_NETWORK.md) - specifically set manual speed and turn off auto-negotiation +for the port used to connect U55C card and connect card to switch. + +### Running Jungfraujoch software +Main Jungfraujoch service is called `jfjoch_broker`. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. +It is recommended to run the service as `systemd` service. + +`jfjoch_broker` takes two parameters: JSON configuration file and HTTP port (default is 5232). +Example JSON files are placed in `etc/` folder. JSON file format is also explained in the OpenAPI definition, as `jfjoch_settings` data structure. + +When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring. + +Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one `jfjoch_broker` on a single machine, each confined to a disjoint subset of GPUs, set `CUDA_VISIBLE_DEVICES`; setting `CUDA_DEVICE_ORDER=PCI_BUS_ID` keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host: +``` +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232 +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233 +``` + +To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. +These need to be obtained from the PSI Detector Group. + +### Card verification + +To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_fpga_test` tool. +For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs: +``` +jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 +``` +Or 1M pixel system with one FPGA card: +``` +jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000 +``` + +## Install Jungfraujoch writer +Jungfraujoch writer is an additional service, that can connect to `jfjoch_broker` ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard. + +At the moment it is better to have a separate machine, with access to distributed file system, for writing images. + +Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands: +``` +mkdir build +cd build +cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch +``` + +## Install Jungfraujoch image viewer +Jungfraujoch viewer is X-ray diffraction image viewer, that is optimized to open Jungfraujoch HDF5 files. + +The viewer is a Qt application and it requires recent version of the library, therefore it is an optional dependency. + +To include it in the building of Jungfraujoch use `-DJFJOCH_VIEWER_BUILD=ON` directive for CMake: +``` +mkdir build +cd build +cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch +``` + + +## Install Jungfraujoch Python client +Use pip: +```shell +pip install jfjoch-client +``` \ No newline at end of file diff --git a/_sources/DETECTORS.md.txt b/_sources/DETECTORS.md.txt new file mode 100644 index 00000000..a4d56b76 --- /dev/null +++ b/_sources/DETECTORS.md.txt @@ -0,0 +1,15 @@ +# Supported detectors + +## PSI detectors +Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfruajoch controls the detector via statically compiled `slsDetectorPackage` into its source code. +It is important that detector firmware has to match `slsDetectorPackage` version used in Jungfraujoch. +The default is 8.0.2; 9.2.0 is built with the `SLS9=ON` CMake option and published in the `slsdet9` +[package repositories](REPOSITORIES.md). +See [PSI Detector group website](https://www.psi.ch/en/lxn/software-releases) for details. + +# DECTRIS detectors + +Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. +In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles output data stream of the DCU. +This mode, called "lite" mode, doesn't use FPGA boards, but mostly CPUs and GPUs for indexing. +The mode is currently experimental and intended for low data rates (100 Hz). diff --git a/_sources/DETECTOR_GEOMETRY.md.txt b/_sources/DETECTOR_GEOMETRY.md.txt new file mode 100644 index 00000000..aec81e9e --- /dev/null +++ b/_sources/DETECTOR_GEOMETRY.md.txt @@ -0,0 +1,51 @@ +# Detector geometry + +At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location +vs. detector frame. It is not recommended to place detector modules stacked. + +The simplest case is detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance +and wavelength. + +For more complex case, one can provide tilt of the detector rotation in PyFAI convention. +This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, +where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector +is defined with three rotation angles: `rot1` (rotating detector right), `rot2` (rotating detector downwards), `rot3` (rotating detector clockwise). +See [PyFAI documentation](https://pyfai.readthedocs.io/en/stable/) for more details. + +## What a pixel coordinate means: (0, 0) is the centre of the first pixel + +Pixel coordinates in Jungfraujoch and rugnux are **0-based and pixel-centred**: an integer coordinate +is the *centre* of that pixel, so pixel *i* covers [*i* − 0.5, *i* + 0.5) and the sensor spans +−0.5 … width − 0.5. A beam centre of 948.0 × 546.0 sits in the middle of pixel [546][948], not on any +of its corners; 948.5 is the boundary between pixel 948 and 949. + +This holds throughout the code: spot and reflection centroids are intensity-weighted sums of the +integer pixel indices, the resolution and azimuthal-bin maps evaluate pixel (col, row) at exactly +(col, row), and a fractional coordinate is turned back into a pixel index by rounding, not by +truncation. The same convention applies to every coordinate the system exposes — the beam centre +(`beam_x_pxl`/`beam_y_pxl` in the API and broker configuration, `--beam-x`/`--beam-y` in rugnux, +`beam_center_x`/`beam_center_y` in NXmx and in the CBOR stream), the spot and predicted-reflection +positions written to HDF5, and the PONI reported by `--mode calibration`. + +Other programs place the origin differently, and the difference is worth half a pixel — enough to +matter when a geometry is copied between programs and then refined: + +| Convention | Beam centre equivalent to our *x* = 948.0 | +|---|---| +| Jungfraujoch, rugnux | 948.0 | +| XDS (`ORGX`/`ORGY`) | 949.0 — also pixel-centred, but pixels are numbered from 1 | +| Measured from the edge of the sensor, in length units — pyFAI (`Poni1`/`Poni2`), DIALS/dxtbx | (948.0 + 0.5) × pixel size, because the centre of pixel *i* is at (*i* + 0.5) × pixel size from the edge | + +The `.poni` file written by `rugnux --mode calibration` is in pyFAI's frame and so already carries +that half pixel; the pixel values the same run reports are ours. + +## Macromolecular crystallography convention for the vertical direction +One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, +with Y values increasing downwards. This is also consistent with computer image formats. + +However, other techniques (as well as internal operation of PSI X-ray detectors) might follow convention, for point (0,0) +being in the bottom left corner and Y values increasing upwards. Such a convention is used, for example, by PyFAI. + +In general, convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows mirroring detector in Y. + +Extra care has to be taken by the user to ensure that no errors are made. \ No newline at end of file diff --git a/_sources/FPGA.md.txt b/_sources/FPGA.md.txt new file mode 100644 index 00000000..a92e413b --- /dev/null +++ b/_sources/FPGA.md.txt @@ -0,0 +1,82 @@ +# FPGA smartNIC + +See separate document for [installation instructions](DEPLOYMENT.md). + +## Hardware +Currently supported FPGA is only **Xilinx Alveo U55C**. + +See AMD/Xilinx webpage for [card user guide (UG1469)](https://docs.xilinx.com/r/en-US/ug1469-alveo-u55c). +According to the user guide: +``` +Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling. +``` + +Card needs to be placed in PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. +There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. +Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C. + +Two variants of the card are available: +* `100g` - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch. +* `8x10g` - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables +are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch. + +See [network documentation](FPGA_NETWORK.md) for details of network. + +## Building firmware +The firmware build targets are generated by CMake only when `vivado` and `vitis_hls` are detected in +the path, and the Vivado version has to match the one below precisely. + +### Xilinx Vivado +The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version **2022.2** installed on the machine. +Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - +specifically newer versions of Vivado will not work. + +In additional to Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary: +* Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html), to build `100g` design. +* Paid 10G/25G Subsystem for Ultrascale+ to build `8x10g` design. +PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams +generated by PSI continuous integration pipeline for `8x10g` is only allowed for non-commercial use. +### HLS compilation +Make HLS routines: +``` +mkdir build +cd build +cmake .. +make hls +``` + +### Synthesis +Create PCIe `100g` bitstream with the following command: +``` +mkdir build +cd build +cmake .. +make pcie_100g +``` +and `8x10g`: +``` +mkdir build +cd build +cmake .. +make pcie_8x10g +``` +### When Vivado is not present + +During CMake execution, the following executables: `vivado` and `vitis_hls` must be present in the path. +If not, build targets will not be generated, and such or similar error message will show up: +``` +$ make pcie_100g +make: *** No rule to make target 'pcie_100g'. Stop. +``` + +### Firmware releases +The firmware is stable and is carried from version to version: the MCS files attached to a release +are normally the ones from the release before it (see [Release contents](RELEASE_CONTENTS.md)). When +it does need to change, it is rebuilt with the targets above on a machine with Vivado. + +### Frame generator + +Jungfraujoch card is equipped with frame generator. It allows to simulate JUNGFRAU detector without having access to such system. +It is placed in parallel to Ethernet MAC - so it is placed before the network stack and before any processing happening on the card. +In the future a redirection will be possible to send the simulated stream through the 100G TX network link. +Frame generator is written in HLS and controlled with AXI-Lite. \ No newline at end of file diff --git a/_sources/FPGA_DATA_ANALYSIS.md.txt b/_sources/FPGA_DATA_ANALYSIS.md.txt new file mode 100644 index 00000000..a392f0b4 --- /dev/null +++ b/_sources/FPGA_DATA_ANALYSIS.md.txt @@ -0,0 +1,83 @@ +# FPGA data analysis + +Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities. + +## Pixel mask +Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. +According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) +and will be excluded from a subsequent analysis. + +## ADU histogram +Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature +of unconverted values. This is done on a module-basis and works with bins with 32 ADU width. + +For EIGER this can be used as just a histogram procedure. + +## JUNGFRAU conversion +For JUNGFRAU module images are converted from ADUs to energy value and divided by a given number to keV units. +Result of the operation is rounded to integers. + +## Pixel thresholding +Pixel range can be specified. +Pixels below a minimum threshold will be assigned zero. +Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). +This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled. + +## Frame summation +Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together. + +## Azimuthal integration +To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. +This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. +Corrections were implemented according to formulas developed by [Jensen et al. (J. Synchr. Rad., 29, 1420-1428, 2022)](https://journals.iucr.org/s/issues/2022/06/00/fv5148/). + +Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited as 2048 per detector module. +This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. +One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit pixed point integer implementation +and is tuned for standard SAXS/WAXS range. + +As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, +so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows. + +## Spot finding +Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels: +1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run). +2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding, +3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation, +4. Pixel value - pixels above certain threshold value can be marked as strong, +5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong, +6. Connected pixels - strong pixels can be discarded if they are "alone", so their 8 directly neighboring pixels are not counted as strong pixels. + +While besides bad pixels criterion, all the above are optional (can be turned off), only pixels that fulfill all enabled criteria are selected as strong pixels. + +### SNR ratio calculation +Signal-to-noise ratio is calculated for a rectangular area. +In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. +This is dictated by the data flow within the FPGA. +In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. +Given very large box size, approximation are made, for example that `N ≈ N-1` in calculating standard deviation. + +## Region-of-interest (ROI) integration +There are 16 ROIs, and the ROI map holds a 16-bit mask per pixel, so a pixel can belong to any subset of them (including none). For each ROI, sum, sum of squares, max count, and number of valid pixels will be calculated. +Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis. + +ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal +(Q-range with an optional φ-sector) — are also evaluated on CPU by the shared `image_analysis/roi/` +engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for +detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path. + +## Pixel statisitics +The following statistics are collected for each module: +* Number of masked pixels +* Number of saturated pixels (excl.masked) +* Number of error pixels (excl. masked) +* Sum of valid pixels in the module +* Minimum value of valid pixels in the module +* Maximum value of valid pixels in the module +Valid pixels are not masked, not saturated, not error pixels. + +## Square root compression +Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. +Scheme was described in [Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020](https://doi.org/10.1107/S1600576720001776). +Pixel value `X` is replaced with `round(sqrt(N*N*X))`, i.e. `round(N*sqrt(X))`, where `N` is integer constant in range 1 to 16. +`N` is what the host writes to the `sqrtmult` register; the FPGA squares it before multiplying the pixel value. diff --git a/_sources/FPGA_DESIGN.md.txt b/_sources/FPGA_DESIGN.md.txt new file mode 100644 index 00000000..56cd1c19 --- /dev/null +++ b/_sources/FPGA_DESIGN.md.txt @@ -0,0 +1,21 @@ +# FPGA data flow + +The following steps are performed on FPGA (in the order of operation): + +1. UDP header decoding +2. SLS detector header decoding +3. State machine that controls data acquisition (start/stop/cancel) +4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules +5. ADU histogram for JUNGFRAU +6. Mask pixels from missing packets with special value +7. Reorder lines for EIGER to form a proper module +8. Mask pixels based on provided pixel mask +9. JUNGFRAU conversion with gain and pedestal corrections +10. Threshold to zero pixels below certain count value +11. Integration according to predefined map (e.g., 1D azimuthal integration) +12. Spot finding +13. ROI calculation +14. Image lossy compression using N*sqrt(pixel) values +15. Send images, analysis results and metadata to host memory via PCI Express + +Each step has dedicated core, written in the high-level synthesis. Exact operation of cores for data analysis is explained in dedicated [document](FPGA_DATA_ANALYSIS.md). \ No newline at end of file diff --git a/_sources/FPGA_LICENSE.md.txt b/_sources/FPGA_LICENSE.md.txt new file mode 100644 index 00000000..540a9605 --- /dev/null +++ b/_sources/FPGA_LICENSE.md.txt @@ -0,0 +1,295 @@ +# FPGA license + +FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. +The license is equivalent of GNU Public License with adaptations for hardware. +See [OHL webpage](https://ohwr.org/project/cernohl/-/wikis/Documents/CERN-OHL-version-2) for details and FAQs. + +## CERN Open Hardware Licence Version 2 - Strongly Reciprocal + + +Preamble + +CERN has developed this licence to promote collaboration among +hardware designers and to provide a legal tool which supports the +freedom to use, study, modify, share and distribute hardware designs +and products based on those designs. Version 2 of the CERN Open +Hardware Licence comes in three variants: CERN-OHL-P (permissive); and +two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this +licence, CERN-OHL-S (strongly reciprocal). + +The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in +unmodified form only. + +Use of this Licence does not imply any endorsement by CERN of any +Licensor or their designs nor does it imply any involvement by CERN in +their development. + + +1 Definitions + +1.1 'Licence' means this CERN-OHL-S. + +1.2 'Compatible Licence' means + +a) any earlier version of the CERN Open Hardware licence, or + +b) any version of the CERN-OHL-S, or + +c) any licence which permits You to treat the Source to which + it applies as licensed under CERN-OHL-S provided that on + Conveyance of any such Source, or any associated Product You + treat the Source in question as being licensed under + CERN-OHL-S. + +1.3 'Source' means information such as design materials or digital +code which can be applied to Make or test a Product or to +prepare a Product for use, Conveyance or sale, regardless of its +medium or how it is expressed. It may include Notices. + +1.4 'Covered Source' means Source that is explicitly made available +under this Licence. + +1.5 'Product' means any device, component, work or physical object, +whether in finished or intermediate form, arising from the use, +application or processing of Covered Source. + +1.6 'Make' means to create or configure something, whether by +manufacture, assembly, compiling, loading or applying Covered +Source or another Product or otherwise. + +1.7 'Available Component' means any part, sub-assembly, library or +code which: + +a) is licensed to You as Complete Source under a Compatible + Licence; or + +b) is available, at the time a Product or the Source containing + it is first Conveyed, to You and any other prospective + licensees + +i) as a physical part with sufficient rights and + information (including any configuration and + programming files and information about its + characteristics and interfaces) to enable it either to + be Made itself, or to be sourced and used to Make the + Product; or +ii) as part of the normal distribution of a tool used to + design or Make the Product. + +1.8 'Complete Source' means the set of all Source necessary to Make +a Product, in the preferred form for making modifications, +including necessary installation and interfacing information +both for the Product, and for any included Available Components. +If the format is proprietary, it must also be made available in +a format (if the proprietary tool can create it) which is +viewable with a tool available to potential licensees and +licensed under a licence approved by the Free Software +Foundation or the Open Source Initiative. Complete Source need +not include the Source of any Available Component, provided that +You include in the Complete Source sufficient information to +enable a recipient to Make or source and use the Available +Component to Make the Product. + +1.9 'Source Location' means a location where a Licensor has placed +Covered Source, and which that Licensor reasonably believes will +remain easily accessible for at least three years for anyone to +obtain a digital copy. + +1.10 'Notice' means copyright, acknowledgement and trademark notices, +Source Location references, modification notices (subsection +3.3(b)) and all notices that refer to this Licence and to the +disclaimer of warranties that are included in the Covered +Source. + +1.11 'Licensee' or 'You' means any person exercising rights under +this Licence. + +1.12 'Licensor' means a natural or legal person who creates or +modifies Covered Source. A person may be a Licensee and a +Licensor at the same time. + +1.13 'Convey' means to communicate to the public or distribute. + + +2 Applicability + +2.1 This Licence governs the use, copying, modification, Conveying +of Covered Source and Products, and the Making of Products. By +exercising any right granted under this Licence, You irrevocably +accept these terms and conditions. + +2.2 This Licence is granted by the Licensor directly to You, and +shall apply worldwide and without limitation in time. + +2.3 You shall not attempt to restrict by contract or otherwise the +rights granted under this Licence to other Licensees. + +2.4 This Licence is not intended to restrict fair use, fair dealing, +or any other similar right. + + +3 Copying, Modifying and Conveying Covered Source + +3.1 You may copy and Convey verbatim copies of Covered Source, in +any medium, provided You retain all Notices. + +3.2 You may modify Covered Source, other than Notices, provided that +You irrevocably undertake to make that modified Covered Source +available from a Source Location should You Convey a Product in +circumstances where the recipient does not otherwise receive a +copy of the modified Covered Source. In each case subsection 3.3 +shall apply. + + You may only delete Notices if they are no longer applicable to + the corresponding Covered Source as modified by You and You may + add additional Notices applicable to Your modifications. + Including Covered Source in a larger work is modifying the + Covered Source, and the larger work becomes modified Covered + Source. + +3.3 You may Convey modified Covered Source (with the effect that You +shall also become a Licensor) provided that You: + +a) retain Notices as required in subsection 3.2; + +b) add a Notice to the modified Covered Source stating that You + have modified it, with the date and brief description of how + You have modified it; + +c) add a Source Location Notice for the modified Covered Source + if You Convey in circumstances where the recipient does not + otherwise receive a copy of the modified Covered Source; and + +d) license the modified Covered Source under the terms and + conditions of this Licence (or, as set out in subsection + 8.3, a later version, if permitted by the licence of the + original Covered Source). Such modified Covered Source must + be licensed as a whole, but excluding Available Components + contained in it, which remain licensed under their own + applicable licences. + + +4 Making and Conveying Products + +You may Make Products, and/or Convey them, provided that You either +provide each recipient with a copy of the Complete Source or ensure +that each recipient is notified of the Source Location of the Complete +Source. That Complete Source is Covered Source, and You must +accordingly satisfy Your obligations set out in subsection 3.3. If +specified in a Notice, the Product must visibly and securely display +the Source Location on it or its packaging or documentation in the +manner specified in that Notice. + + +5 Research and Development + +You may Convey Covered Source, modified Covered Source or Products to +a legal entity carrying out development, testing or quality assurance +work on Your behalf provided that the work is performed on terms which +prevent the entity from both using the Source or Products for its own +internal purposes and Conveying the Source or Products or any +modifications to them to any person other than You. Any modifications +made by the entity shall be deemed to be made by You pursuant to +subsection 3.2. + + +6 DISCLAIMER AND LIABILITY + +6.1 DISCLAIMER OF WARRANTY -- The Covered Source and any Products +are provided 'as is' and any express or implied warranties, +including, but not limited to, implied warranties of +merchantability, of satisfactory quality, non-infringement of +third party rights, and fitness for a particular purpose or use +are disclaimed in respect of any Source or Product to the +maximum extent permitted by law. The Licensor makes no +representation that any Source or Product does not or will not +infringe any patent, copyright, trade secret or other +proprietary right. The entire risk as to the use, quality, and +performance of any Source or Product shall be with You and not +the Licensor. This disclaimer of warranty is an essential part +of this Licence and a condition for the grant of any rights +granted under this Licence. + +6.2 EXCLUSION AND LIMITATION OF LIABILITY -- The Licensor shall, to +the maximum extent permitted by law, have no liability for +direct, indirect, special, incidental, consequential, exemplary, +punitive or other damages of any character including, without +limitation, procurement of substitute goods or services, loss of +use, data or profits, or business interruption, however caused +and on any theory of contract, warranty, tort (including +negligence), product liability or otherwise, arising in any way +in relation to the Covered Source, modified Covered Source +and/or the Making or Conveyance of a Product, even if advised of +the possibility of such damages, and You shall hold the +Licensor(s) free and harmless from any liability, costs, +damages, fees and expenses, including claims by third parties, +in relation to such use. + + +7 Patents + +7.1 Subject to the terms and conditions of this Licence, each +Licensor hereby grants to You a perpetual, worldwide, +non-exclusive, no-charge, royalty-free, irrevocable (except as +stated in subsections 7.2 and 8.4) patent licence to Make, have +Made, use, offer to sell, sell, import, and otherwise transfer +the Covered Source and Products, where such licence applies only +to those patent claims licensable by such Licensor that are +necessarily infringed by exercising rights under the Covered +Source as Conveyed by that Licensor. + +7.2 If You institute patent litigation against any entity (including +a cross-claim or counterclaim in a lawsuit) alleging that the +Covered Source or a Product constitutes direct or contributory +patent infringement, or You seek any declaration that a patent +licensed to You under this Licence is invalid or unenforceable +then any rights granted to You under this Licence shall +terminate as of the date such process is initiated. + + +8 General + +8.1 If any provisions of this Licence are or subsequently become +invalid or unenforceable for any reason, the remaining +provisions shall remain effective. + +8.2 You shall not use any of the name (including acronyms and +abbreviations), image, or logo by which the Licensor or CERN is +known, except where needed to comply with section 3, or where +the use is otherwise allowed by law. Any such permitted use +shall be factual and shall not be made so as to suggest any kind +of endorsement or implication of involvement by the Licensor or +its personnel. + +8.3 CERN may publish updated versions and variants of this Licence +which it considers to be in the spirit of this version, but may +differ in detail to address new problems or concerns. New +versions will be published with a unique version number and a +variant identifier specifying the variant. If the Licensor has +specified that a given variant applies to the Covered Source +without specifying a version, You may treat that Covered Source +as being released under any version of the CERN-OHL with that +variant. If no variant is specified, the Covered Source shall be +treated as being released under CERN-OHL-S. The Licensor may +also specify that the Covered Source is subject to a specific +version of the CERN-OHL or any later version in which case You +may apply this or any later version of CERN-OHL with the same +variant identifier published by CERN. + +8.4 This Licence shall terminate with immediate effect if You fail +to comply with any of its terms and conditions. + +8.5 However, if You cease all breaches of this Licence, then Your +Licence from any Licensor is reinstated unless such Licensor has +terminated this Licence by giving You, while You remain in +breach, a notice specifying the breach and requiring You to cure +it within 30 days, and You have failed to come into compliance +in all material respects by the end of the 30 day period. Should +You repeat the breach after receipt of a cure notice and +subsequent reinstatement, this Licence will terminate +immediately and permanently. Section 6 shall continue to apply +after any termination. + +8.6 This Licence shall not be enforceable except by a Licensor +acting as such, and third party beneficiary rights are +specifically excluded. \ No newline at end of file diff --git a/_sources/FPGA_NETWORK.md.txt b/_sources/FPGA_NETWORK.md.txt new file mode 100644 index 00000000..cad5e8d4 --- /dev/null +++ b/_sources/FPGA_NETWORK.md.txt @@ -0,0 +1,39 @@ +# FPGA network + +The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 is lower port (when PCIe connector is on the bottom). +The card FPGA design is offered in two variants `100g` and `8x10g`. These have different behavior regarding the network: + +`100g` this variant operates QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a **switch**. +QSFP28 transceivers are necessary. + +`8x10g` this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables + are necessary. It is designed for **detector directly connected** to the Jungfraujoch server, without switch. + +## Transceivers +AMD doesn't provide transceiver compatibility matrix for Alveo U55C. +In our experience operating the card we haven't seen issues with transceivers from various providers (FS.com, Mellanox, Finnisar). +We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn't +support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven't +encountered such a situation. + +## Switch configuration +Special care has to be taken for switch operation, given the FPGA core doesn't support auto-negotiation. It is necessary to configure switch port +to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000). + +## Network LEDs +Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). +LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. +For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs means all ports connected, orange LEDs at least one of the ports connected. + +## Network stack +Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. +Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed. + +The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. +The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. +The card won't respond to broadcast ICMP pings. + +Each link can be put in `direct` mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. +This settings should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any +10 Gbit/s link on the same card. Currently `direct` mode is turned OFF for `100g` design and ON for `8x10g` design. +This can be manually adjusted for each link. \ No newline at end of file diff --git a/_sources/FPGA_PCIE_DRIVER.md.txt b/_sources/FPGA_PCIE_DRIVER.md.txt new file mode 100644 index 00000000..743d7f24 --- /dev/null +++ b/_sources/FPGA_PCIE_DRIVER.md.txt @@ -0,0 +1,100 @@ +# FPGA PCIe driver + +## Compilation +To compile kernel module type: +``` +make +``` + +## Installation +To install kernel module, you need to have root permissions and run: +``` +sudo make install +``` + +## Loading driver into kernel +After installing the kernel driver, it should be possible to insert it into the kernel via: +``` +modprobe jfjoch +``` + +## Ownership of the character devices +By default, character devices `/dev/jfjoch` are owned by root (user/group) and are not accessible by others. +This means that `jfjoch_broker` must be running as superuser, which might not be optimal for security reasons in most cases. +The behavior can be changed by creating `udev` rules. Create a file called `/etc/udev/rules.d/99-jfjoch.rules` +with the following content: +``` +KERNEL=="jfjoch*" OWNER="" GROUP="" +``` +It is OK to provide only group, for example to make the devices accessible by group `jungfrau`: +``` +KERNEL=="jfjoch*" GROUP="jungfrau" +``` + +## DKMS +To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel +module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. +For other systems, it is necessary to follow the procedure below, though it is not well tested. + +This first requires to install DKMS - for RHEL it is available via EPEL repository: +``` +sudo dnf install dkms +``` +Then use script provided in the driver directory to copy driver code to DKMS directory: +``` +./install_dkms.sh +``` +If upgrading the driver, please first remove current driver from DKMS system: +``` +dkms remove jfjoch -v --all +``` + +## Driver parameters +Currently, there is one driver parameter `nbuffers`, that defines count of exchange buffers (see below). +This can be adjusted in the modprobe operation, for example: +``` +modprobe jfjoch nbuffers=1024 +``` + +## Exchange buffers +The parameter defines number of buffers used to exchange data between card and host application. +Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. +These buffers are used by both card-to-host and host-to-card operations. + +Buffers use special allocation, as they are continuous in physical address space, which helps the FPGA card to transfer all +data associated with detector module in two DMA transfers (one data, one metadata). +Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in power of two, **4 MiB** is safe number for one buffer size. +Buffer can be mapped into user space, but performing `mmap` system call on the `/dev/jfjoch` character device. + +Buffer count can be adjusted by setting `nbuffers` parameter. There are two considerations for setting optimal value: +1. For card-to-host transfers, minimal value is roughly +` * `, +this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64. +2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. +So minimal count is ` * (3 + 3 * )`. + +Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration. + +## Known problems +To avoid inconsistent behavior, this driver won't load if release number differs between the kernel driver and FPGA card. + +## CMake file +While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. +It is not made for actual compilation of the kernel driver and should not be used for that purpose. + +## Character device access +For each FPGA device a character device is created called `/dev/jfjoch`. +When device is opened two operations are possible: +mmap() to map exchange buffers +ioctl() to communicate with the cards +Interfacing should be done through the JungfraujochDevice class in `fpga/host_library` directory. + +## Sysfs access +Certain performance counters can be read through sysfs mechanism in the kernel. +One needs to `cat` files in `/sys/class/misc/jfjoch/` directory. + +## RHEL 9.5+ issue +RedHat Enterprise Linux 9.5 backported modification to settings virtual memory flags from Linux kernel 6.3, while still operating kernel version 5.14. +It is complicated to come up with a single rule to select when newer functions should be used, so it works with RHEL 9.5+, +while still being compatible with other Linux distributions. It is even more complex given not all RHEL compatible distributions adopted the change at the same version. +For the moment the quick fix is to define an environment variable `HAVE_VM_FLAGS_SET` before making the kernel. diff --git a/_sources/FPGA_SETTINGS.md.txt b/_sources/FPGA_SETTINGS.md.txt new file mode 100644 index 00000000..b5e7bce0 --- /dev/null +++ b/_sources/FPGA_SETTINGS.md.txt @@ -0,0 +1,121 @@ +# FPGA advanced reference +## Register map +FPGA setup can be done via registers: + +| Address | Bits | Meaning | Mode | Notes | +|---------------------|------|------------------------------------------------------------------------------------------------|:-----|----------------------------------------------| +| 0x000000 - 0x00FFFF | | Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory) | | | +| 0x010000 | 32 | Action Control Register | | | +| | | Bit 0 - Action start | R/W | | +| | | Bit 1 - Action idle | R | | +| | | Bit 2 - Action cancel | R/W | cleared on reset or action start | +| | | Bit 3 - Clear network counters | R/W | cleared on reset | +| | | Bit 12:4 - Debug signals (see action_config.v for details) | R | | +| | | Bit 16 - AXI Mailbox interrupt 0 | R | | +| 0x010004 | 32 | Reserved | - | | +| 0x010008 | 32 | Reserved | - | | +| 0x01000C | 32 | GIT SHA1 | R | | +| 0x010010 | 32 | Reserved | R | | +| 0x010014 | 32 | Reserved | R | | +| 0x010018 | 32 | Jungfraujoch FPGA variant | R | | +| 0x01001C | 32 | Reserved | R | | +| 0x010020 | 32 | Max. number supported detector modules | R | constant | +| 0x010024 | 32 | Reserved | R | constant | +| 0x010028 | 64 | Pipeline stalls before writing to host memory | R | reset on action start | +| 0x010030 | 64 | Pipeline stalls before accessing HBM | R | reset on action start | +| 0x010038 | 32 | FIFO status (see action_config.v for details) | R | | +| 0x01003C | 32 | Size of single HBM channel in bytes (default value for the particular card) | R/W | should not be altered for standard operation | +| 0x010040 | 64 | Packets processed by the action | R | cleared on reset or action start | +| 0x010048 | 64 | Valid ethernet packets | R | cleared on reset | +| 0x010050 | 64 | Valid ICMP packets | R | cleared on reset | +| 0x010058 | 64 | Valid UDP packets | R | cleared on reset | +| 0x010060 | 64 | Valid detector packets processed by the card | R | cleared on reset | +| 0x010068 | 64 | Packets flagged as errors by CMAC | R | cleared on reset | +| 0x010070 | 64 | Pipeline stalls before data processing | R | reset on action start | +| 0x010078 | 64 | AXI-beats before accessing HBM | R | reset on action start | +| 0x010080 | 64 | AXI-beats before data processing | R | reset on action start | +| 0x010088 | 64 | AXI-beats before host writer | R | reset on action start | +| 0x010090 | 64 | Last encountered SwissFEL pulse ID | R | cleared on reset | +| 0x010100 | 32 | Spot finder photon count threshold | R/W | | +| 0x010104 | 32 | Spot finder signal-to-noise ratio threshold (single-precision float) | R/W | | +| 0x010200 | 64 | MAC address source for internal frame generator | R/W | network byte order | +| 0x010208 | 32 | IPv4 address source for internal frame generator | R/W | network byte order | +| 0x01020C | 32 | Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.) | R/W | | +| 0x010210 | 32 | Data collection mode | R/W | | +| | | Bit 0 - Conversion to photons | | | +| | | Bit 1 - Output extend to 32-bit | | | +| | | Bit 2 - Output is unsigned integer | | | +| | | Bit 3 - Use sq. root lossy compression | | | +| | | Bit 7 - JUNGFRAU fixed G1 mode | | | +| | | Bit 8 - Set to zero values below threshold | | | +| | | Bit 16:31 - Data collection ID (carried with completions) | | | +| 0x010214 | 32 | Photon energy in keV (single-precision float) | R/W | | +| 0x010218 | 32 | Number of frames expected in the data collection (defines termination condition) | R/W | | +| 0x01021C | 32 | Number of storage cells | R/W | | +| 0x010220 | 32 | Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.) | R/W | | +| 0x010224 | 32 | Coefficient for sq. root compression (need to set bit in data collection mode to apply) | R/W | | +| 0x010225 | 32 | Threshold; set values below set to zero (need to set bit in data collection mode to apply) | R/W | | +| 0x030000 - 0x03FFFF | | AXI Mailbox for Work Request / Work Completion | | See Xilinx PG114 for register map | +| 0x040000 - 0x04FFFF | | QuadSPI flash | | See Xilinx PG153 for register map | +| 0x050000 - 0x05FFFF | | Interrupt controller | | See Xilinx PG099 for register map | +| 0x060000 - 0x06FFFF | | Load calibration (HLS) | | | +| 0x070000 - 0x07FFFF | | AXI Firewall | | See Xilinx PG293 for register map | +| 0x080000 - 0x08FFFF | | Frame generator (HLS) | | | +| 0x090000 - 0x09FFFF | | PCIe DMA control | | See Xilinx PG195 for register map | +| 0x0A0000 - 0x0AFFFF | | I2C clock generator | | See Xilinx PG195 for register map | +| 0x0C0000 - 0x0FFFFF | | Xilinx Card Management Solution Subsystem management subsystem | | See Xilinx PG348 for register map | +| 0x100000 - 0x10FFFF | | MAC 10G / CMAC 100G | | See Xilinx PG210/PG203 for register map | +| 0x110000 - 0x11FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x120000 - 0x12FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x130000 - 0x13FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x140000 - 0x14FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x150000 - 0x15FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x160000 - 0x16FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x170000 - 0x17FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x200000 - 0x20FFFF | | Eth/IPv4 network stack for interface #0 | | | +| 0x210000 - 0x21FFFF | | Eth/IPv4 network stack for interface #1 | | | +| 0x220000 - 0x22FFFF | | Eth/IPv4 network stack for interface #2 | | | +| 0x230000 - 0x23FFFF | | Eth/IPv4 network stack for interface #3 | | | +| 0x240000 - 0x24FFFF | | Eth/IPv4 network stack for interface #4 | | | +| 0x250000 - 0x25FFFF | | Eth/IPv4 network stack for interface #5 | | | +| 0x260000 - 0x26FFFF | | Eth/IPv4 network stack for interface #6 | | | +| 0x270000 - 0x27FFFF | | Eth/IPv4 network stack for interface #7 | | | +| 0x400000 - 0x47FFFF | 64 | Address table: decodes handles used by load_calibration and host_writer to DMA addresses | | | + +## AXI Mailbox + +AXI mailbox is used to send work request from host to action, and receive work completions. +Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114). + +Work request has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------------------------| +| 0 | 15 | Work request ID (handle) | + +Work completion has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------| +| 0 | 15 | Work request ID (handle) | +| | | Special values: | +| | | 65534 - start of data collection | +| | | 65535 - end of data collection | +| 15 | 31 | Data collection ID | + +## HBM memory + +| Interface number | Core | Meaning | +|------------------|------------------|---------------------------------------------| +| 0-1 | jf_conversion | Gain factor G0 | +| 2-3 | jf_conversion | Gain factor G1 | +| 4-5 | jf_conversion | Gain factor G2 | +| 6-7 | jf_conversion | Pedestal G0 | +| 8-9 | jf_conversion | Pedestal G1 | +| 10-11 | jf_conversion | Pedestal G2 | +| 12-13 | integration | Integration map | +| 14-15 | integration | Integration weights | +| 16-17 | spot_finder_mask | Spot finder resolution | +| 18-19 | roi_calc | ROI calculation | +| 20-21 | frame_generator | Frame generator | +| 22-27 | load_from_hbm | Frame summation | diff --git a/_sources/HARDWARE.md.txt b/_sources/HARDWARE.md.txt new file mode 100644 index 00000000..527c2844 --- /dev/null +++ b/_sources/HARDWARE.md.txt @@ -0,0 +1,55 @@ +# Hardware requirements +Operating Jungfraujoch requires the following: + +1. High performance server +2. FPGA board(s) installed in the server +3. (optionally) GPU boards +4. (optionally) 100G switch to connect FPGA and the detector + +Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include +AMD FPGA boards. Therefore, the two has to be purchases separately. This might have impact on the warranty for the hardware +and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn't take any responsibility +for warranty limitations for operating FPGA boards in the server. Having said this - we didn't encounter any hardware issues so far. + +## High performance server +PSI is using HPE DL380 Gen11 servers are the moment to operate Jungfraujoch systems. However, this is because of general +preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system +from any other vendor with similar specification should work as well. + +At PSI, we use the following configuration of HPE DL380 Gen11 to operate 9M pixel detectors at 2 kHz is as follows: +* 2 x Intel Xeon 8558P +* 512 GB RAM +* 2 x Nvidia L4 GPU (for indexing) +* 1 x Nvidia Connect-X 6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs) +* Copper 1G/10G network + +### PCI slots +When ordering the system it is important to ensure enough PCIe cards can be accommodated in the system. +In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2x GPU, 1x network + +Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK. + +## FPGA +Jungfraujoch is built for [AMD/Xilinx U55C](https://www.amd.com/en/products/accelerators/alveo/u55c/a-u55c-p00g-pq-g.html) +(A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported. + +Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. +For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires +using TX delay functionality in the detector. + +## GPUs +Operating fast-feedback indexer code requires operation of a graphic processing unit from Nvidia. +For practical reasons, i.e. power consumption and cost, we choose inference grade card Nvidia L4. +In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work. + +## Network switch +Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs `8x10g` variant +of the Jungfraujoch FPGA image, which allows to directly connect 4 JUNGFRAU modules to one U55C card. + +Such configuration is however +impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujochs server. +In this case one needs a network switch. + +We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. +For switches with only 100G ports it is important to ensure, that these can be split into 4x10G ports to connect the detector. + diff --git a/_sources/HDF5.md.txt b/_sources/HDF5.md.txt new file mode 100644 index 00000000..fad40276 --- /dev/null +++ b/_sources/HDF5.md.txt @@ -0,0 +1,488 @@ +# HDF5 / NeXus data format + +Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html)-compliant. On top of the +NXmx application definition, Jungfraujoch records a substantial amount of *derived* metadata +(spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These +extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and +reusable. + +This page documents the **file layout and the data fields**. The operational behaviour of the +writer (running, republishing, file finalisation) is described in +[jfjoch_writer](JFJOCH_WRITER.md). The wire format that feeds the writer is described in +[CBOR messages](CBOR.md); fields below frequently correspond one-to-one to CBOR message fields, and +that document is a useful companion for their meaning. + +## 1. Motivation: derived metadata and FAIR data + +The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep +them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both +**rotation** macromolecular crystallography (single- and multi-crystal, including fine-sliced and +helical scans) and **serial** crystallography (stills, grid scans); the same concerns apply to both: + +* **Findability.** Raw diffraction images carry almost no descriptive metadata about *content*. + Quantities such as background level, number of diffraction spots, or indexing outcome let a user + judge the quality and relevance of a dataset *before* inspecting the raw images. +* **Accessibility at scale.** A single experiment can span tens to hundreds of terabytes. Standard + retrieval (e.g. HTTP) makes a dataset *available* but not *inspectable* — users would otherwise + have to download a large fraction of the data just to decide whether it is useful. Compact + derived representations make discovery, assessment and reuse feasible. + +Because Jungfraujoch couples acquisition with real-time analysis used to *steer* experiments, +transparency and reproducibility of that analysis matter. As a minimum the writer therefore +preserves spot-finding and indexing results together with the filters that were applied, and it can +retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse. + +### Two complementary layouts: per-image spots vs. a reflection table + +Jungfraujoch stores analysis products in two shapes, matching how each is accessed. + +**Per-image spot finding / indexing.** Spot finding and indexing are inherently *image-centric* — +the natural query is "give me the spots for image *n*" — and this holds for serial stills and for +rotation frames alike. For these products Jungfraujoch adopts a layout similar to the +[Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the convention +understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties (position, +intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, +with each image allocated room for up to a predefined maximum number of spots. These dense arrays +are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved +without traversing variable-length structures. The cost is some storage overhead for unused slots +(padded with sentinels), which is acceptable for the access pattern. + +**Integrated reflections.** Integrated intensities are naturally a *dataset-wide* table, which is +exactly the model of the NeXus +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class. +This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration +results (see §4.2 below). We deliberately do *not* force spot finding/indexing into a single +experiment-wide table: across the hundreds of thousands of patterns typical of serial — or +fine-sliced rotation — experiments, that would require aggregating the whole experiment before the +spots of one image can be read. We encourage the community to develop standardised NeXus application +definitions for image-centric crystallography products that combine NeXus interoperability with the +access patterns and scale of modern high-throughput experiments. + +## 2. File layout + +A run is written as one **master file** plus, depending on the format, one or more **data files**: + +``` +_master.h5 # NXmx master file (metadata + links / virtual datasets) +_data_000001.h5 # data file: images + per-image analysis +_data_000002.h5 +... +``` + +The master file is produced by `writer/HDF5NXmx.cpp`; data files by `writer/HDF5DataFile.cpp` and +its plugins (`writer/HDF5DataFilePlugin*.cpp`). Files are written to a temporary `*..tmp` +name and renamed on successful close. + +Three master-file variants exist (set via `file_format`): + +| Format | Value | Master ↔ data linking | +|--------|:-----:|------------------------| +| **NXmxLegacy** (default) | 1 | One external link in `/entry/data` per data file (`data_000001`, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0. | +| **NXmxVDS** | 2 | A single virtual dataset `/entry/data/data` spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+. | +| **NXmxIntegrated** | 3 | No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format. | + +In legacy/VDS mode, image-indexed analysis arrays live in the **data files** and are exposed in the +master file through external links or virtual datasets; in integrated mode they are written +directly into the single file. Throughout this document a "✓ in master" column marks entries that +are visible (directly or via link/VDS) from the master file. + +Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or +bitshuffle + Zstd; signed integer image datasets use `INTx_MIN` as the HDF5 fill value (the +"masked / no-data" sentinel), unsigned use `UINTx_MAX`. + +### Reprocessing output: `_process.h5` + +The offline reprocessing tool [`rugnux`](TOOLS.md) (`rugnux/rugnux_cli.cpp`) re-runs the +full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing +dataset and writes its results to a master file named **`_process.h5`**. This file uses the +**integrated** format, but instead of copying the images its `/entry/data/data` is a *virtual +dataset that links back to the original image files* (`hdf5_source_data` → +`NXmx::LinkToData_ProcessingVDS`). The result is a compact, self-describing companion file that +holds *all* the derived analysis (everything in §4) plus a virtual view +of the raw images — without duplicating terabytes of data. + +This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead +of) the raw data to convey what is in a dataset and how it processed, while the `/entry/data/data` +VDS still resolves to the original images when they are available. `rugnux` can also process +an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. + +## 3. NXmx-standard content + +The entries below are part of, or valid base classes for, the +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html) application definition. +"NXmx" = listed in the application definition; "base" = a valid field of the relevant NeXus base +class (`NXdetector`, `NXsample`, `NXsource`) but not in the NXmx required/recommended subset. + +### `/entry` (NXentry) + +| Field | Std | Notes | +|-------|:---:|-------| +| `definition` | NXmx | value `"NXmx"` | +| `start_time` | NXmx | arming time | +| `end_time`, `end_time_estimated` | NXmx | approximate end time | + +File-level HDF5 attributes `file_name`, `file_time`, `HDF5_Version` are also set. + +### `/entry/source` (NXsource), `/entry/instrument` (NXinstrument) + +| Field | Std | Units | +|-------|:---:|-------| +| `source/name`, `source/type` | NXmx / base | | +| `source/current` | base | A | +| `instrument/name` | NXmx | | + +### `/entry/instrument/beam` (NXbeam) + +| Field | Std | Units | +|-------|:---:|-------| +| `incident_wavelength` | NXmx | angstrom | +| `incident_wavelength_spread` | NXmx | angstrom (only if polychromatic) | +| `total_flux` | NXmx | Hz | + +### `/entry/instrument/attenuator` (NXattenuator) + +| Field | Std | +|-------|:---:| +| `attenuator_transmission` | NXmx | + +### `/entry/instrument/detector` (NXdetector) + +| Field | Std | Units | +|-------|:---:|-------| +| `depends_on` | NXmx | → `transformations/rot3` | +| `beam_center_x`, `beam_center_y` | NXmx | pixel (0.0 = centre of the first pixel, see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)) | +| `distance` | NXmx | m | +| `count_time`, `frame_time` | NXmx | s | +| `sensor_thickness` | NXmx | m | +| `sensor_material` | NXmx | | +| `description` | NXmx | | +| `threshold_energy` | NXmx | eV (EIGER; written only for a single channel) | +| `x_pixel_size`, `y_pixel_size` | base | m | +| `serial_number` | base | | +| `bit_depth_readout` | NXmx | | +| `saturation_value` | NXmx | | +| `flatfield_applied` | NXmx | | +| `pixel_mask`, `pixel_mask_applied` | NXmx | `pixel_mask` is `[y, x]`, hard-linked from `detectorSpecific/pixel_mask` | +| `countrate_correction_applied` | NXmx | | +| `number_of_cycles` | base | frame-summation factor | + +### `/entry/instrument/detector/transformations` (NXtransformations) + +The NXtransformations *mechanism* (the `depends_on` chain, `transformation_type`, `vector`, +`offset` attributes) is standard. The axis **names** follow the PyFAI PONI convention chosen by +Jungfraujoch (see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)): + +| Axis | Type | Units | Depends on | +|------|------|-------|-----------| +| `translation` | translation | m | `.` | +| `rot1` | rotation | rad | `translation` | +| `rot2` | rotation | rad | `rot1` | +| `rot3` | rotation | rad | `rot2` | + +The beam centre is encoded in `translation` (its offset from the sample), not only in the +informational `beam_center_x`/`beam_center_y` fields. In a `_process.h5` written by rugnux these axes +carry the **refined** detector geometry — the refined beam centre folds into `translation` and the +refined tilt into `rot1`/`rot2`/`rot3`; the broker writes the user-provided geometry unchanged. + +### `/entry/instrument/detector/module` (NXdetector_module) + +`data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all +NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). + +### `/entry/sample` (NXsample) + +| Field | Std | Units / notes | +|-------|:---:|-------| +| `name` | NXmx | | +| `depends_on` | NXmx | points at the last goniometer / grid-scan axis, or `.` for stills | +| `temperature` | NXmx | K | +| `transformations/` (NXtransformations) | NXmx | rotation axis (e.g. `omega`) or grid-scan translation; hard-linked as `/entry/sample/goniometer` | +| `unit_cell` | base | `[a, b, c, α, β, γ]` | +| `ub_matrix` | base | `[1, 3, 3]`, Angstrom⁻¹ | + +For a rotation scan the goniometer axis is written as a per-image angle array `` plus +`_end`, scalar `_range_average`, `_range_total`, and for helical scans +`_helical_x/_y/_z`. These extra goniometer datasets beyond the bare axis array are Jungfraujoch +conveniences. + +### `/entry/data` (NXdata) + +`data` (3-D image stack, `[n_images, y, x]`) with `image_nr_low` / `image_nr_high` attributes. +In legacy mode this group instead contains one external link `data_000001`, … per data file. + +## 4. Extensions beyond NXmx + +Everything in this section is **outside the NXmx standard**. Each group is declared with +`NX_class = NXcollection` (the NeXus-sanctioned container for non-standardised content) unless noted. +The per-image arrays are indexed by image number, padded to the run length and filled with a +sentinel (`NaN` for floats, `-1`/`0` for integer indices) where a quantity is absent. + +### 4.1 `/entry/MX` — spot finding and indexing (CXI-style) + +The flagship extension. Spot ("peak") properties are stored as fixed-size `[n_images, max_spots]` +arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as `[n_images]` vectors. +In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master. + +**Per-spot arrays `[n_images, max_spots]`:** + +| Dataset | Units | Meaning | Indexing only | +|---------|-------|---------|:---:| +| `peakXPosRaw`, `peakYPosRaw` | pixel | spot position (raw detector frame) | | +| `peakTotalIntensity` | photons | spot intensity | | +| `peakIceRingRes` | | spot lies in an ice-ring resolution band | | +| `peakH`, `peakK`, `peakL` | | Miller indices of the (indexed) spot | ✓ | +| `peakDistEwaldSphere` | Å⁻¹ | distance of the spot from the Ewald sphere | ✓ | +| `peakIndexed` | | spot fits the indexing solution | ✓ | +| `peakLattice` | | lattice the spot belongs to (`-1` = unindexed) | ✓ | + +**Per-image vectors `[n_images]`:** + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `nPeaks` | | number of spots stored for the image (CXI) | +| `strongPixels` | | strong-pixel count (first spot-finding stage) | +| `peakCountUnfiltered` | | spots found before filtering | +| `peakCountLowRes` | | low-resolution spots | +| `peakCountIceRingRes` | | spots inside ice-ring bands | +| `peakCountIceRingControl` | | spots in the ice-free flanks beside those bands, rescaled to their q width - the control for the count above (their ratio, pooled over the run, is the spot-based ice indicator) | +| `peakCountIndexed` | | spots fitting the indexing solution | +| `imageIndexed` | | image was indexed (0/1) | +| `indexingLatticeCount` | | number of lattices found for the image | +| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A* (2016), Vol. A, [Table 3.1.3.1](https://onlinelibrary.wiley.com/iucr/itc/Ac/ch3o1v0001/table3o1o3o1.pdf)) | +| `bravaisLattice` | | Bravais lattice short code, e.g. `aP`, `mC`, `oF`, `tI`, `hP`, `hR`, `cF` | +| `profileRadius` | Å⁻¹ | crystal profile radius | +| `mosaicity` | deg | mosaicity estimate | +| `bFactor` | Ų | per-image B-factor estimate | +| `resolutionEstimate` | Å | diffraction resolution estimate | +| `integratedReflections` | | number of integrated reflections | +| `bkgEstimate` | photons | mean background in the 3–5 Å resolution band | +| `iceRingScore` | ratio | strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | +| `beam_corr_x`, `beam_corr_y` | pixel | beam-center correction applied during processing | +| `imageScaleFactor` | | on-the-fly per-image scale factor *g* | +| `imageScaleCC` | | on-the-fly scaling correlation coefficient | +| `imageScaleMosaicity` | deg | scaling-model mosaicity | +| `sweepQuality` | | why this image's stretch of the sweep was flagged — see below | + +**Per-image lattices:** `latticeIndexed` `[n_images, 9]` (Å) — the real-space lattice (flattened +3×3); `latticeIndexedExtra` `[n_images, max_extra_lattices, 9]` (Å) — additional orientation +variants. + +**Run-level summaries** (written into the master `/entry/MX` at finalisation): + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `indexing_algorithm` | | `FFBIDX` / `FFT (CUDA)` / `FFT (FFTW)` | +| `geom_refinement_algorithm` | | e.g. `beam_center` | +| `rotationLatticeIndexed` | Å | whole-run rotation-indexing lattice (`[9]`) | +| `rotationLatticeIndexedExtra` | Å | additional whole-run lattices (`[m, 9]`) | +| `rotationLatticeNiggliClass` | | Niggli class of the run lattice | +| `imageIndexedMean` | | mean indexing rate over the run | +| `bkgEstimateMean` | photons | mean background over the run | +| `iceRingScoreMean` | ratio | mean `iceRingScore` over the run — the single "how icy was this dataset" number (1 = no ice) | +| `indexedLatticeCount` | | per-image lattice count summary (master). *Note: data files use `indexingLatticeCount`; readers accept either.* | + +**Sweep quality.** `sweepQuality` `[n_images]` (`uint8`) says why the stretch of the sweep this +image belongs to was flagged as delivering much less than the rest of the run: **0** means it was +not, and any other value is a **1-based index into `sweepQualityReasons`**, a string vector written +beside it that carries the whole vocabulary, so the codes can be read without this source. The +vocabulary is closed and stable — a code is never renamed and never reused — and currently reads +`no_diffraction`, `crystal_out_of_beam`, `weak_diffraction`, `loss_of_centring`, `radiation_damage`; +[the rugnux documentation](RUGNUX.md#sweep-quality-and-the-reason-vocabulary) defines what each one +means. Both datasets are **absent** unless the sweep-quality diagnostic ran, which needs scaling and +merging; their absence therefore means "not looked for", *not* "every image clean". Written by the +offline `rugnux` path only — the broker does not merge — and not carried on the CBOR stream, in the +same way as the other offline-only fields (`space_group_number`, the refined geometry). Nothing is +excluded from processing on the strength of it. The condensed, dataset-wide form of the same finding +is in `_report.txt`. + +CrystFEL can read the spots directly with: + +``` +peak_list = /entry/MX +peak_list_type = cxi +``` + +### 4.2 `/entry/reflections` — integrated reflections (NXreflections) + +Integrated reflections are stored **per image** as +`/entry/reflections/image_NNNNNN` groups, each declared `NX_class = NXreflections`. The columns map +mostly onto the standard +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class: + +| Dataset | Units | NXreflections | Meaning | +|---------|-------|:-------------:|---------| +| `h`, `k`, `l` | | standard | Miller indices | +| `d` | Å | standard | resolution | +| `int_sum` | photons | standard | integrated intensity (summation) | +| `int_err` | photons | non-standard name | σ of the intensity (standard equivalent: `int_sum_errors`) | +| `background_mean` | photons | standard | mean background under the peak | +| `background_variance` | photons² | non-standard | non-signal part of σ², carried to the merge. Absent in files written before it existed; the reader then recovers it from σ² − I | +| `predicted_x`, `predicted_y` | pixel | name standard, units differ | predicted position. NXreflections `predicted_x/_y` are *physical* lengths; the pixel datasets are `predicted_px_x/_y` | +| `observed_x`, `observed_y` | pixel | name standard, units differ | observed centroid (pixels; standard pixel form is `observed_px_x/_y`) | +| `observed_frame` | | standard | image number of the reflection | +| `lp` | | standard | Lorentz–polarization factor (stored as `1/rlp`) | +| `partiality` | | standard | recorded fraction of the reflection | +| `delta_phi` | deg | **extension** | XDS Δφ: offset from the centre of the current frame | +| `zeta` | | **extension** | Lorentz ζ factor (reciprocal-space geometry term) | +| `image_scale_corr` | | **extension** | per-image scale correction; `I_true = image_scale_corr · int_sum` | + +In the master file these per-image groups are exposed through `/entry/reflections` external links +(VDS/integrated formats). + +### 4.3 `/entry/azint` — azimuthal integration + +| Dataset | Shape | Units | Meaning | +|---------|-------|-------|---------| +| `bin_to_q` | `[φ_bins, q_bins]` | Å⁻¹ | q value of each bin | +| `bin_to_two_theta` | `[φ_bins, q_bins]` | deg | 2θ of each bin | +| `bin_to_phi` | `[φ_bins, q_bins]` | deg | azimuthal angle of each bin | +| `image` | `[n_images, φ_bins, q_bins]` | | per-image integrated profile (NaN for empty bins) | +| `image_std` | `[n_images, φ_bins, q_bins]` | | per-bin standard deviation | +| `image_count` | `[n_images, φ_bins, q_bins]` | | pixels contributing per bin | +| `map` | `[y, x]` | | pixel→bin mapping (master file only) | + +### 4.4 `/entry/roi` — regions of interest (per-image results) + +`/entry/roi/` has one sub-group per configured ROI, holding the **per-image result +vectors** `[n_images]`. These are written into the data files; in VDS mode they are exposed from +the master file through virtual datasets, and in integrated mode they are in the single file. +(In legacy mode they remain only in the data files.) + +| Dataset | Meaning | +|---------|---------| +| `max` | maximum pixel value in the ROI | +| `sum` | sum of pixel values | +| `sum_sq` | sum of squared pixel values | +| `npixel` | number of valid pixels | +| `x`, `y` | intensity-weighted centroid | + +### 4.4.1 `/entry/roi_defs` — ROI definitions (master file) + +The **dataset-wide ROI definitions** (geometry, fixed for the whole acquisition) live in the +master file under a *separate* `/entry/roi_defs` group — kept apart from `/entry/roi` above so +that older readers, which iterate `/entry/roi`, are unaffected by these entries. One sub-group +`/entry/roi_defs/` per ROI: + +| Dataset | Meaning | +|---------|---------| +| `bit_index` | which bit of `roi_map` (below) marks this ROI | +| `type` | `box`, `circle` or `azim` | +| `min_x_pxl`, `max_x_pxl`, `min_y_pxl`, `max_y_pxl` | box bounds (type `box`) | +| `center_x_pxl`, `center_y_pxl`, `radius_pxl` | circle (type `circle`) | +| `q_min_recipA`, `q_max_recipA` | Q range (type `azim`) | +| `phi_min_deg`, `phi_max_deg` | azimuthal-angle sector (type `azim`, omitted for a full ring) | + +`/entry/roi_defs/roi_map` `[y, x]` is a `uint16` per-pixel bitmask: bit `bit_index` is set for +every pixel belonging to that ROI, so an ROI's footprint can be recovered exactly. + +### 4.5 `/entry/image` — per-image pixel statistics + +`[n_images]` vectors: `max_value`, `min_value` (viable min/max, excluding error/saturated pixels), +`error_pixels`, `saturated_pixels`, `pixel_sum`. Surfaced in the master file under `/entry/image`. + +### 4.6 `/entry/profiling` — per-image timing + +`[n_images]` vectors in seconds: `spotFindingTime`, `indexingTime`, `integrationTime`, +`refinementTime`, `processingTime`, `braggPredictionTime`, `preprocessingTime`, `compressionTime`, +`azIntTime`, `indexAnalysisTime`, `imageScaleTime`. + +### 4.7 `/entry/detector` — acquisition diagnostics (data file) + +A convenience NXcollection in the data file (note: distinct from the standard +`/entry/instrument/detector`). In **integrated** format these datasets are written under +`/entry/instrument/detector/detectorSpecific` instead. + +| Dataset | Meaning | +|---------|---------| +| `timestamp`, `exptime` | per-image timestamp and exposure time | +| `number` | image number (original number if image rejection was used) | +| `det_info` | JUNGFRAU debug field | +| `storage_cell_image` | storage-cell number | +| `rcv_delay`, `rcv_free_send_buffers` | receiver internal diagnostics | +| `packets_expected`, `packets_received` | UDP packets per image | +| `data_collection_efficiency_image` | received / expected packet ratio | + +### 4.8 `/entry/xfel` — pulsed-source metadata + +`[n_images]` vectors `pulseID` and `eventCode`, written for pulsed sources (e.g. SwissFEL). + +### 4.9 Other collections + +| Path | Class | Content | +|------|-------|---------| +| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode`, `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/`, `data_collection_efficiency_image` | +| `/entry/instrument/detector/calibration` | NXcollection | per-channel pedestal / calibration images (bitshuffle-compressed) | +| `/entry/instrument/fluorescence` | NXcollection | XRF spectrum: `energy` [eV], `data` | +| `/entry/user` | NXcollection | scalar values supplied under `header_appendix.hdf5` | + +### 4.10 Non-standard fields inside the NXmx detector group + +A few extension scalars are written *inside* the otherwise-standard `/entry/instrument/detector` +group for compatibility with existing tooling: + +| Field | Units | Meaning | +|-------|-------|---------| +| `detector_distance` | m | duplicate of `distance` (Dectris/Neggia compatibility) | +| `detector_number` | | detector identifier (Dectris convention) | +| `error_value` | | masked/error pixel sentinel (NXmx standard would be `underload_value`) | +| `bit_depth_image` | | stored image bit depth (NXmx standard is `bit_depth_readout`) | +| `acquisition_type` | | always `triggered` (Dectris convention) | +| `jungfrau_conversion_applied` | | JUNGFRAU photon/keV conversion applied | +| `jungfrau_conversion_factor` | eV | conversion factor | +| `geometry_transformation_applied` | | module→full-detector geometry applied | + +### 4.11 User-supplied metadata: `header_appendix` and `image_appendix` + +Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two +free-form JSON fields in the `/start` request (`broker/jfjoch_api.yaml`) provide this without any +schema change; both accept *any valid JSON*: + +| Field | Carried in | Persisted to HDF5? | +|-------|-----------|--------------------| +| `header_appendix` | the **start** message, under `user_data.user` (see [CBOR](CBOR.md)) | no — except the `hdf5` sub-object (below) | +| `image_appendix` | **every image** message, as `user_data` | no | + +Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, +republished analysis, viewers), so they are the recommended channel for facility- or +beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has +no dedicated API field. + +**Persisting selected values to HDF5.** `header_appendix` is normally *not* written to the master +file. As an exception, if it contains a key `hdf5` whose value is a JSON object of scalars (strings +and numbers — no arrays or nested objects), the writer stores each entry under `/entry/user/`. + +For example, a `/start` request containing: + +```json +{ + "header_appendix": { + "proposal": "p20001", + "operator": "jdoe", + "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 } + }, + "image_appendix": { "trigger_source": "external" } +} +``` + +forwards the whole `header_appendix` as `user_data.user` on the start message and +`{"trigger_source": "external"}` as `user_data` on every image message, and writes three scalars +into the master file: + +``` +/entry/user/beamline = "X06SA" +/entry/user/ring_mode = "top-up" +/entry/user/attenuator_foils = 2 +``` + +## 5. Notes + +* **Units** are written as the HDF5 `units` attribute on the dataset (e.g. `m`, `eV`, `deg`, + `Angstrom`, `Angstrom^-1`, `Angstrom^2`, `pixel`, `s`). +* **Sentinels.** Missing per-image values are `NaN` (floats) or `-1`/`0` (integer indices); image + pixels use `INTx_MIN` / `UINTx_MAX`. +* **Master vs data file.** In legacy/VDS formats the analysis arrays physically live in the data + files; the master file links to them (external links in legacy, virtual datasets in VDS). In the + integrated format there are no data files and everything is in one place. +* **CXI / CrystFEL.** `/entry/MX` follows the CXI peak-list convention; see + [CXI file format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf). diff --git a/_sources/IMAGE_STREAM.md.txt b/_sources/IMAGE_STREAM.md.txt new file mode 100644 index 00000000..f445eb92 --- /dev/null +++ b/_sources/IMAGE_STREAM.md.txt @@ -0,0 +1,235 @@ + +# Data streams + +Jungfraujoch process (`jfjoch_broker`) operates three outputs. +All three can be operated/enabled independently. +These are: +* **Image** - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket) +* **Preview** - images with metadata at a reduced frame rate (PUB socket) +* **Metadata** - only metadata for all the images, bundled into packages (PUB socket) + +## Image stream +Images (with metadata) are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message), [calibration messages](CBOR.md#calibration-message) and [end message](CBOR.md#end-message) with run metadata. + +If `file_prefix` is not provided for a data collection, images won't be sent to image stream (or its HDF5/CBOR replacements). + +### Splitting image stream +Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. +All sockets will forward start and end messages. Only first socket will forward calibration messages and will be marked to write master file. + +### ZeroMQ image stream +This is using PUSH ZeroMQ socket(s). +It should be strictly avoided to have multiple receivers connected to one PUSH ZeroMQ socket. +ZeroMQ will send the images in a round-robin basis to the receivers. +In this case start and end messages will end up only with one receiver. +Instead, Jungfraujoch feature of multiple sockets should be used. +For ZeroMQ image stream, each writer connects to a different port. + +Behavior is as following: +* Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer. +* Calibration message is sent to the first socket only, with timeout of 1s. +* Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped). +* End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded. + +The format is generally interchangeable with DECTRIS Stream2 format. + +#### ZeroMQ configuration + +ZeroMQ image stream is configured in the broker JSON configuration file under the `zeromq_settings` section: +```json +{ + "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"], + "send_watermark": 100, + "send_buffer_size": 67108864, + "writer_notification_socket": "tcp://192.168.0.1:*" +} +``` + +- `image_socket`: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (`tcp://`, `ipc://`). `0.0.0.0` binds on all network interfaces. +- `send_watermark` (optional): ZeroMQ send high-water mark (number of outstanding messages per socket). +- `send_buffer_size` (optional): OS-level send buffer size for the ZeroMQ socket. +- `writer_notification_socket` (optional): see [Writer notification socket](#writer-notification-socket) below. + +### TCP/IP image stream +This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. +This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication +between the data collection and the writer, and is therefore more robust than ZeroMQ. + +For TCP/IP image stream, Jungfraujoch **listens** on a single TCP port and all writers **connect** to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic `KEEPALIVE` frames when no data collection is active to detect dead connections; writers are expected to respond with a `KEEPALIVE` pong. + +Using `*` as port number (e.g. `tcp://127.0.0.1:*`) is supported — the OS assigns a free port and the actual bound address can be queried via `GetAddress()`. + +Payloads for `START`, `DATA`, `CALIBRATION` and `END` frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages. +`ACK`, `CANCEL`, and `KEEPALIVE` are control frames (no CBOR payload). + +The data collection lifecycle on each connection follows: +`START` → `CALIBRATION` (socket 0 only) → `DATA` (repeated) → `END` + +If a `START` ACK fails on any connection, Jungfraujoch sends `CANCEL` to all already-started connections and rolls back. + +For each frame: +1. Read one `TcpFrameHeader` (fixed size, 64-byte aligned). +2. Validate `magic` (`0x4A464A54` / `"JFJT"`) and `version` (`2`). +3. Read `payload_size` bytes (if non-zero). + +When image stream is split into multiple connections: +- `START` and `END` are sent on all connections, +- `CALIBRATION` is sent only on connection 0, +- `DATA` frames are distributed by file grouping: connection index = `(image_number / images_per_file) % num_connections`. + +#### TCP/IP configuration + +TCP/IP image stream is configured in the broker JSON configuration file under the `tcp_settings` section: +```json +{ + "image_socket": "tcp://192.168.0.1:9100", + "nwriters": 2, + "send_buffer_size": 67108864 +} +``` + +- `addr`: listen address in `tcp://:` format. `0.0.0.0` binds on all interfaces. `*` as port selects a random free port. +- `nwriters` (optional): maximum number of simultaneous writer connections accepted. +- `send_buffer_size` (optional): OS-level `SO_SNDBUF` size for accepted connections. + +#### ACK handling + +ACK handling is mandatory for correct operation: +- `START` **must** be acknowledged (`ACK` with `ack_for=START`) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered. +- `END` **must** be acknowledged (`ack_for=END`) on each connection within 10 seconds for successful completion. +- `CANCEL` should be acknowledged during rollback paths (500ms timeout). +- `DATA` should be acknowledged for every frame. A `DATA` ACK with `FATAL` flag set reports a downstream error (e.g. disk full) which is propagated to `jfjoch_broker` via `Finalize()`. A failed `DATA` ACK does **not** break the TCP connection on its own — data continues to flow. +- `CALIBRATION` is not acknowledged at this time. +- `KEEPALIVE` frames are not acknowledged via ACK; the writer responds with a `KEEPALIVE` pong frame instead. + +#### Keepalive + +When no data collection is active, Jungfraujoch sends `KEEPALIVE` frames approximately every 5 seconds on each persistent connection. Writers should respond with a `KEEPALIVE` frame (pong). OS-level TCP keepalive is also enabled (`TCP_KEEPIDLE=30s`, `TCP_KEEPINTVL=10s`, `TCP_KEEPCNT=3`) as a secondary safety net. Dead connections are automatically removed from the pool. + +#### Zero-copy transmission + +On Linux, large payload transmission (`DATA` and `CALIBRATION` frames) can use kernel TCP zero-copy (`SO_ZEROCOPY`/`MSG_ZEROCOPY`) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal `send()` behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread. + +#### Frame types + +| Value | Name | Purpose | +|---:|---|---| +| 1 | `START` | Start-of-run metadata | +| 2 | `DATA` | One image payload | +| 3 | `CALIBRATION` | Calibration payload | +| 4 | `END` | End-of-run metadata | +| 5 | `ACK` | Acknowledgement / error reporting | +| 6 | `CANCEL` | Cancel run initialization/stream | +| 7 | `KEEPALIVE` | Connection liveness probe/pong | + +#### TCP frame header (`TcpFrameHeader`) + +| Field | Type | Description | +|--------------------------|---|----------------------------------------------------------| +| `magic` | `uint32_t` | Protocol magic (`0x4A464A54`, `"JFJT"`) | +| `version` | `uint16_t` | Protocol version (`2`) | +| `type` | `uint16_t` | Frame type (see table above) | +| `image_number` | `uint64_t` | Image index for `DATA` frames | +| `payload_size` | `uint64_t` | Number of payload bytes after header | +| `socket_number` | `uint32_t` | Connection index in split-stream mode | +| `flags` | `uint32_t` | ACK flags (`OK`, `FATAL`, `HAS_ERROR_TEXT`) | +| `run_number` | `uint64_t` | Run identifier | +| `ack_processed_images` | `uint32_t` | In `ACK`: number of images processed by receiver | +| `ack_code` | `uint16_t` | In `ACK`: error/status code | +| `ack_for` | `uint16_t` | In `ACK`: frame type being acknowledged | +| `ack_fifo_occupancy` | `uint16_t` | In `ACK`: occupancy of input FIFO in the `jfjoch_writer` | +| `ack_fifo_max_occupancy` | `uint64_t` | In `ACK`: max occupancy of input FIFO | + +The header is 64-byte aligned (`alignas(64)`). + +#### ACK semantics + +- `ACK` frames use `ack_for` to indicate which frame type is acknowledged. +- `flags`: + - `OK` (bit 0): operation accepted/successful, + - `FATAL` (bit 1): receiver reports unrecoverable error (primarily for `DATA`), + - `HAS_ERROR_TEXT` (bit 2): ACK payload contains UTF-8 error text. +- `ack_code` can be used to categorize errors: + +| Code | Name | Meaning | +|---:|---|---| +| 0 | `None` | No error | +| 1 | `StartFailed` | START processing failed | +| 2 | `DataWriteFailed` | Image write failed | +| 3 | `EndFailed` | END processing failed | +| 4 | `DiskQuotaExceeded` | Disk quota exceeded | +| 5 | `NoSpaceLeft` | No space left on device | +| 6 | `PermissionDenied` | Permission denied | +| 7 | `IoError` | General I/O error | +| 8 | `ProtocolError` | Protocol-level error | + +### Image stream replacement +Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting "None" image pusher for all the measurements. + +## Writer notification socket +The writer notification socket is used **only with ZeroMQ image stream**. Since ZeroMQ is asynchronous, `jfjoch_broker` does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back. + +For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame. + +To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with `writer_notification_socket` entry: +```json +{ + "writer_notification_socket":"tcp://192.168.0.1:*" +} +``` +Such entry will create PULL socket on `192.168.0.1` network interface listening on one, random TCP port. When data processing is started, the +image stream will send CBOR [start message](CBOR.md#start-message). This message will include information on `writer_notification_zmq_addr`, +which needs to be used by downstream code. Since the start message must reference the address of `jfjoch_broker` host, notification +socket should always listen on a particular network interface, and should not be configured with placeholder address `0.0.0.0`. It is, however, OK +to use placeholder `:*` for network port, as it will be substituted for the one chosen by ZeroMQ. + +For every image stream socket, downstream code must send the following message to the PULL socket: +```json +{ + "run_number":135, + "run_name": "sample_1", + "socket_number": 1, + "processed_images":250, + "ok": true +} +``` +Here `run_number`, `run_name` and `socket_number` must match information from the start message. +`ok` is boolean confirming if the writing process was OK. +`processed_images` is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. +If not, it is possible to include error message: +```json +{ + "run_number":135, + "run_name": "sample_1", + "socket_number": 1, + "processed_images": 0, + "ok": false, + "error": "Permission error" +} +``` +This way errors from the downstream code are propagated to `jfjoch_broker`. + +If writer notification socket is configured, but downstream code doesn't send proper notification, `jfjoch_broker` will time out after 60 seconds producing an error message. + +## Preview stream +Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. +Images are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. +Only start and image messages are sent. + +This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope +with the messages, it will always receive the last generated message (no backlog). +For this reason it is also recommended to use the same option on receiver side. + +Given PUB socket properties, it is possible to connect multiple viewers to a single socket --- all the viewers should receive all the images sent. + +## Metadata stream +Jungfraujoch can also send pure metadata for the purpose of archiving such information. +Metadata are serialized as CBOR [metadata message](CBOR.md#metadata-message). +This is very similar as image message, but excludes the actual image array and spot positions. +As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. +Order of images within bundle, as well a size of the bundle, are not guaranteed. +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. + +This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected. \ No newline at end of file diff --git a/_sources/JFJOCH_BROKER.md.txt b/_sources/JFJOCH_BROKER.md.txt new file mode 100644 index 00000000..833f8868 --- /dev/null +++ b/_sources/JFJOCH_BROKER.md.txt @@ -0,0 +1,188 @@ +# jfjoch_broker + +`jfjoch_broker` is the main service for the Jungfraujoch application. It is responsible for: + +* Providing user interface via HTTP and OpenAPI +* Configuring FPGA firmware +* Building images from FPGA output and forwarding the results over ZeroMQ + +## External interfaces +Broker operates four external interfaces. + +**Image stream** ZeroMQ PULL socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream +processing. See details [here](IMAGE_STREAM.md#image-stream). + +**Preview stream** ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details [here](IMAGE_STREAM.md#preview-stream). + +**Metadata stream** ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details [here](IMAGE_STREAM.md#metadata-stream). + +**Configuration, status and results interface** HTTP/REST interface described in the OpenAPI format. +Description of the API is presented in the [OpenAPI specification](OPENAPI_SPECS.rst). + +## Broker configuration +`jfjoch_broker` requires JSON configuration files. The file is described by OpenAPI structure `jfjoch_settings` defined in `jfjoch_api.yaml` file. +It is recommended to go through example files in the `etc/`. + +Example with all fields: + +```json +{ + "pcie": [ + { + "blk": "/dev/jfjoch0", + "ipv4": "10.1.1.7" + }, + { + "blk": "/dev/jfjoch1", + "ipv4": "10.1.1.8" + } + ], + "zeromq": { + "send_watermark": 100, + "send_buffer_size": 1024, + "image_socket": [ + "tcp://1.2.3.4:5000", + "tcp://1.2.3.4:5001" + ], + "writer_notification_socket": "tcp://1.3.4.6:7000" + }, + "instrument": { + "source_name": "Swiss Light Source", + "source_type": "Synchrotron X-ray Source", + "instrument_name": "X06SA", + "pulsed_source": false, + "electron_source": false + }, + "detector": [ + { + "description": "EIGER 1M", + "serial_number": "E1M-01", + "type": "EIGER", + "high_voltage_V": 150, + "udp_interface_count": 1, + "module_sync": true, + "sensor_thickness_um": 320, + "calibration_file": [ + "gainMaps.bin" + ], + "hostname": [ + "e1m-01", + "e1m-02" + ], + "readout_time_us": 3, + "sensor_material": "Si", + "tx_delay": [ + 0,1 + ], + "base_data_ipv4_address": "10.10.10.50", + "standard_geometry": { + "nmodules": 1, + "gap_x": 8, + "gap_y": 36, + "modules_in_row": 1 + }, + "custom_geometry": [ + { + "x0": 0, + "y0": 0, + "fast_axis": "Xp", + "slow_axis": "Xp" + } + ], + "mirror_y": true + } + ], + "detector_settings": { + "frame_time_us": 450, + "count_time_us": 0, + "internal_frame_generator": false, + "internal_frame_generator_images": 1, + "detector_trigger_delay_ns": 0, + "timing": "auto", + "eiger_threshold_keV": 6.0, + "jungfrau_pedestal_g0_frames": 2000, + "jungfrau_pedestal_g1_frames": 300, + "jungfrau_pedestal_g2_frames": 300, + "jungfrau_pedestal_g0_rms_limit": 100, + "jungfrau_pedestal_min_image_count": 128, + "jungfrau_storage_cell_count": 1, + "jungfrau_storage_cell_delay_ns": 5000, + "jungfrau_fixed_gain_g1": false, + "jungfrau_use_gain_hg0": false + }, + "azim_int": { + "polarization_factor": -1, + "solid_angle_corr": true, + "high_q_recipA": 0, + "low_q_recipA": 0, + "q_spacing": 0 + }, + "image_format": { + "summation": true, + "geometry_transform": true, + "jungfrau_conversion": true, + "jungfrau_conversion_factor_keV": 0.001, + "bit_depth_image": 16, + "signed_output": true, + "mask_module_edges": true, + "mask_chip_edges": true + }, + "image_buffer_MiB": 2048, + "receiver_threads": 64, + "frontend_directory": "/usr/share/jfjoch/frontend", + "image_pusher": "ZeroMQ", + "zeromq_metadata": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4357" + }, + "zeromq_preview": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4356" + } +} +``` + +## Setting up a local test for Jungfraujoch +For development, it is possible to set up a local installation of Jungfraujoch. +This will work without FPGA installed in the computer and allows to test Jungfraujoch software layer, including +ZeroMQ streaming and file writing. + +The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore +very low, as fixed-point calculations have large performance penalty on CPU. In the CPU simulation mode, one can simulate +using only a single FPGA device. + +To run the test: + +### Compile Jungfraujoch with frontend +``` +mkdir build +cd build +cmake .. +make jfjoch +make frontend +``` +Alternatively, for RHEL8 system, you can use RPM generated by automated pipeline. +Solely `jfjoch` one is enough. +In this case - it is necessary to update `etc/broker_local.json` file with frontend path in `/usr/share/jfjoch/frontend`. + +### Start service +Start broker: +``` +cd build/broker +./jfjoch_broker ../../etc/broker_local.json 5232 +``` + +### Run tests +To run test a Python script is provided: +``` +cd tests/test_data +python jfjoch_broker_test.py +``` +The script will initialize Jungfraujoch, import test image and start data collection. + +### Expected result +You can observe online data analysis by opening the following web page: [http://localhost:5232](http://localhost:5232). +Also, a dataset with images should be written in the `build/broker` directory. + diff --git a/_sources/JFJOCH_VIEWER.md.txt b/_sources/JFJOCH_VIEWER.md.txt new file mode 100644 index 00000000..b355d7de --- /dev/null +++ b/_sources/JFJOCH_VIEWER.md.txt @@ -0,0 +1,154 @@ +# jfjoch_viewer + +`jfjoch_viewer` is the **interactive** desktop application of Jungfraujoch. It opens diffraction +datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal +integration, per-image statistics), and can follow a live data collection by syncing with a +running [`jfjoch_broker`](JFJOCH_BROKER.md) over its HTTP interface. + +It is a standalone Qt 6 application, distributed pre-built for **Linux and Windows** on the Gitea +release page and in the Jungfraujoch RPM/APT repositories — see [Release contents](RELEASE_CONTENTS.md) +for what each package contains and what it requires, and [Deployment](DEPLOYMENT.md) for how to +install it. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| **`jfjoch_viewer`** | **Interactive, on-screen exploration** | **Qt desktop application** | **On screen; a processing job can write the same files as `rugnux`** | +| [`rugnux`](RUGNUX.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | + +## Functionality + +- Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the + `*_process.h5` files produced by [`rugnux`](RUGNUX.md). It also opens NXmx files + written by DECTRIS detectors, though that path has had only limited testing. +- Runs an **embedded data-processing pipeline** — the same analysis code as the rest of + Jungfraujoch — performing spot finding, indexing and integration on the displayed image, with the + result drawn over it. This interactive analysis is not written anywhere. +- Runs **full processing jobs** on the open dataset with *Analyze dataset*, on the same + [`rugnux`](RUGNUX.md) engine and off the GUI thread. The settings panel's **MX / AzInt / Calib** + toggle decides what a run does — full analysis, azimuthal integration only, or a detector + calibration — over a chosen image range, optionally writing `_process.h5` and the merged + `.mtz`/`.cif`. A finished run becomes a selectable view of the dataset, so several processing runs + can be compared against each other, and its merging statistics (or, for a calibration, its fitted + geometry) open in their own window; the *Processing* panel lists the runs and reopens those + results. The equivalent `rugnux` command line can also be copied out to run the same job on a + cluster instead. +- **Detector calibration** against a powder standard, on the *Calib* page: pick the calibrant + (`LaB6`, `AgBh`, `CeO2`, `Si`, `ice`, or the open dataset's own unit cell) and fit either the image + on screen (*Guess* / *Refine detector calibration*) or the whole dataset (*Analyze dataset*, which + writes a pyFAI `.poni`). The whole-dataset fit measures the rings either from the + azimuthally-binned profile summed over the run (*Rings*, the default) or from the pooled spot lists + (*Spots*), and reports PONI x/y, the two tilts and the distance against the header values. Judge it + by the **radial rms**, not the beam-centre sigma: the sigma shrinks with the number of ring points, + so a fit that sits a couple of pixels off every ring can still report a small one. *Rings* needs + the run to be integrated in azimuthal sectors — with the AzInt page's *Azimuthal bins* below 4 the + calibration run raises it to 32, as `rugnux --mode calibration` does, and says so. +- **Settings** panel for the geometry, unit cell, spot finding, indexing, azimuthal integration, + Bragg integration, scaling, powder calibration and a reference dataset — the same settings the + CLI takes. +- Auxiliary windows: image list, dataset metadata, spot list, reflection list, reciprocal-space + viewer, 2D azimuthal-integration image, calibration-image viewer and a magnifier; plus the + *Inspector* (per-image statistics, image features, resolution rings, ROI statistics), the + *Image strip* thumbnail feed and dataset-info charts. +- User-mask editing: build a user mask interactively, load one from TIFF (replacing or adding to the + current one), save it as TIFF, clear it, or upload it to a connected server. +- Layout presets (*View ▸ Image layout / Processing layout / Reset layout*) rearrange the docks for + looking at images or at processing results. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. On systems with a +GPU, use the CUDA build (a separate package variant everywhere: RPM/APT repository, `.tgz` and +Windows installer) for the embedded indexing and integration; the non-CUDA build runs the same +pipeline on the CPU at much lower throughput. The CUDA build also runs on a machine without a GPU — +see [Release contents ▸ CUDA and non-CUDA builds](RELEASE_CONTENTS.md#cuda-and-non-cuda-builds). + +## Opening data + +- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file. +- **File ▸ Open HTTP** (`Ctrl+H`) — connect to a `jfjoch_broker` HTTP endpoint to follow a live + collection. The dialog defaults to host `localhost` and port `8080`; these defaults can be + overridden with the environment variables `JUNGFRAUJOCH_HTTP_HOST` and `JUNGFRAUJOCH_HTTP_PORT`. +- **Command line** — `jfjoch_viewer ` opens a file (or an `http://host:port` URL) on + start-up. `--dbus ` (`-d`) enables or disables the D-Bus interface (default: enabled); + `--help` and `--version` behave as usual. + +## D-Bus interface + +When enabled, the viewer registers the D-Bus interface `ch.psi.jfjoch_viewer`, so other processes +can drive it: + +- `LoadFile(filename, image_number=0, summation=1)` — open a file (or an `http://host:port` URL) + and display the given image. +- `LoadImage(image_number, summation=1)` — navigate to an image in the already-open dataset. + +`summation` sums that many consecutive images before display. + +## Building from source on Windows + +`jfjoch_viewer` is the one Jungfraujoch component that is cross-platform: it builds on Windows 11 +with MSVC and the full CUDA GPU path. (The rest of Jungfraujoch — broker, receiver, FPGA host — is +Linux-only.) A pre-built installer is published with every release, so building from source is only +needed to develop or to change the build options. On Windows the build is automatically restricted +to the viewer and the libraries it needs (`JFJOCH_VIEWER_ONLY` is forced on), and the remaining +dependencies are fetched and built automatically (the first configure needs network access). + +Verified toolchain — the same one the released installer is built with: + +- Windows 11 +- Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC +- CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path +- Qt 6.11 for MSVC (`msvc2022_64`), including the **Qt Charts** module — e.g. `C:\Qt\6.11.1\msvc2022_64` +- CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of + the box — it comes with the C++ workload, so there is nothing extra to install. Any recent + standalone CMake (from cmake.org, or the one bundled with Qt in `C:\Qt\Tools\CMake_64`) works too. +- zlib and Eigen — the two libraries not auto-fetched on Windows. Build/install both into one prefix + (here `C:\deps`) and point CMake at it: + ``` + :: static zlib + git clone --branch v1.3.1 https://github.com/madler/zlib + cmake -G Ninja -S zlib -B zlib-build -DCMAKE_INSTALL_PREFIX=C:/deps + cmake --build zlib-build --target install + :: Eigen 3.4 (header-only) -- install just the headers with `cmake --install`; the BLAS/LAPACK/test + :: targets are disabled since they are not needed (and fail to build under MSVC). Use the 3.4 series: + :: the project requests find_package(Eigen3 3.4), which Eigen's same-major rule rejects for 5.x. + git clone --branch 3.4.0 https://gitlab.com/libeigen/eigen.git + cmake -G Ninja -S eigen -B eigen-build -DCMAKE_INSTALL_PREFIX=C:/deps ^ + -DEIGEN_BUILD_BLAS=OFF -DEIGEN_BUILD_LAPACK=OFF -DEIGEN_BUILD_DOC=OFF -DBUILD_TESTING=OFF + cmake --install eigen-build + ``` +- Optional: [NSIS](https://nsis.sourceforge.io) to build the `.exe` installer. + +Configure and build from an **x64 Native Tools Command Prompt for VS 2026** (so `cl`, `nvcc` and +`ninja` are on `PATH`): + +``` +cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_PREFIX_PATH="C:/deps;C:/Qt/6.11.1/msvc2022_64" +cmake --build build-win --target jfjoch_viewer +``` + +Notes: + +- `CMAKE_PREFIX_PATH` (the `C:/deps` prefix plus Qt) is the only required flag — CMake finds zlib and + Eigen from the prefix, so no separate `-DZLIB_ROOT` is needed. +- The CUDA toolchain is located automatically from the `CUDA_PATH` environment variable that the + CUDA installer sets (or from `nvcc` on `PATH`). Pass `-DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe"` + only if `nvcc` is installed in a nonstandard location and is not found. +- For a machine without an NVIDIA GPU, add `-DJFJOCH_USE_CUDA=OFF`: the viewer then runs the same + pipeline on the CPU (FFTW indexer) at lower throughput. + +To produce a self-contained installer (bundles the Qt runtime via `windeployqt`, the analysis CLIs, +and — on the CUDA build — the cuFFT runtime DLL, so the target host needs neither Qt nor a CUDA +toolkit), with NSIS installed: + +``` +cd build-win +cpack +``` + +The NSIS generator is selected automatically on Windows (no `-G` needed). What comes out, and how +the CUDA and CPU variants are named and told apart, is described in +[Release contents ▸ Windows installer](RELEASE_CONTENTS.md#windows-installer). diff --git a/_sources/JFJOCH_WRITER.md.txt b/_sources/JFJOCH_WRITER.md.txt new file mode 100644 index 00000000..30452e56 --- /dev/null +++ b/_sources/JFJOCH_WRITER.md.txt @@ -0,0 +1,176 @@ +# jfjoch_writer + +`jfjoch_writer` is NeXus compliant HDF5 file writer. + +## Acknowledgements +* Zdenek Matej (MAX IV) +* Felix Engelmann (MAX IV) +for testing and multiple improvement suggestions. + +## Running directory +Writer needs to be running in base directory for writing files - `file_prefix` will be always relative in regard to writer running directory. +Writer detects and protects for basic security issues, like `file_prefix` starting with a slash, or starting with `../`, or containing `/../`. + +## Usage +Writer needs to be started as a background service, with the following command: +``` +jfjoch_writer {options}
+ +Options: +-T | --tcp Use raw TCP/IP instead of ZeroMQ +-j | --nproc= Number of forks (only with -T) +-d | --root_dir= Root directory for file writing (-R is a deprecated alias) +-r | --zmq_repub_port= ZeroMQ port for PUSH socket to republish images +-f | --zmq_file_port= ZeroMQ port for PUB socket for notifications on finalized files +-w | --rcv_watermark= Receiving ZeroMQ socket watermark (default = 100) +-W | --repub_watermark= Republish ZeroMQ socket watermark (default = 1000) +-v | --verbose Verbose output +-h This message +``` +for example: +``` +jfjoch_writer -d /data tcp://dcu-address:5400 +``` + +## Status and cancellation +When a data collection is finalized, each writer reports its outcome back to `jfjoch_broker` over +the writer notification socket — a ZeroMQ address the broker passes in the START message +(`writer_notification_socket` in the broker configuration) — as a JSON message with the socket +number, run name and number, processed image count, throughput, and on failure an error string. +That is how the broker learns that a writer could not write. On the TCP/IP image stream, failures +additionally come back in-band as negative acknowledgements +(see [Data streams](IMAGE_STREAM.md#tcpip-image-stream)). + +To stop a writer, send it `SIGINT`, `SIGQUIT`, `SIGTERM` or `SIGHUP`: it closes the HDF5 files it is +writing and exits. This is only for the case where the broker was terminated or disconnected — it is +not the normal way to end a data collection, which the broker finishes on its own. + +## Republish +Republish creates a PULL socket on the writer, where all the messages are republished for further use by data analysis pipeline. +Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won't be republished. +In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won't be retried. + +Republish functionality is optional, if republish port number is omitted this functionality is not enabled. + +## Overwriting files +When `jfjoch_writer` creates a HDF5 file, it first adds suffix `..tmp`. +Random value depends on current time-stamp and likely will be different from each file of the particular series. +After file is all saved and closed, it is renamed to remove the suffix. +By default, renaming won't happen if this would overwrite existing file. +However, this behavior can be changed by setting `overwrite` parameter to true in the file writer configuration. + +### When the overwrite conflict is reported +An existing output file is a fatal condition (unless `overwrite` is true). *When* it is detected +depends on whether the transport between the broker and the writer has a back-channel to report the +failure before acquisition starts: + +* **Direct HDF5 pusher and TCP writer (back-channel available).** The conflict is detected at + **start**: the writer that owns the master file checks whether it already exists and refuses to + start. The direct pusher raises the error in-process; the TCP writer returns a START-failure + acknowledgement. Either way the broker learns immediately and aborts the data collection *before* + the detector is armed — no images are taken and nothing is written. Only the master file is + checked up front: in a multi-writer setup the per-image data files are staggered across writers, + and checking them at start would make each writer inspect files it never writes (and race the + writers that do). Data-file conflicts are instead caught by their owning writer at the final + rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker. +* **ZeroMQ writer (no back-channel).** The ZeroMQ image stream is fire-and-forget: the writer has no + way to tell the broker to stop, and the broker would keep streaming images regardless. The writer + therefore does **not** fail at start. It writes the whole series to the `..tmp` files as + usual and only fails at the final rename, leaving the `.tmp` files on disk. This is deliberate: the + acquired images are preserved (in `.tmp` form) rather than being dropped by a writer that aborted + mid-stream. Rename the `.tmp` files by hand, or re-run with `overwrite` set, to recover them. + +## Finalized files information +Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure: + +``` +{ + "filename": : HDF5 data file name (relative to writer root directory), + "nimages": number of images in the file (counting from 1!), + "file_number": number of file within the acquisition, + "sample_name": name of sample, + "run_name": name of run, + "run_number": number of run, + "experiment_group": number of p-group / proposal (optional), + "user_data": user_data, + "beam_x_pxl": beam center (X) in pixels, + "beam_y_pxl": beam center (Y) in pixels, + "detector_distance_m": detector distance (X) in m, + "detector_height_pxl": detector size (X) in pixels, + "detector_width_pxl": detector size (Y) in pixels, + "incident_energy_eV": photon energy of the X-ray beam, + "pixel_size_m": pixel size in meter (assuming pixel X == Y), + "saturation": this count and higher mean saturation, + "space_group_number": space group number (optional), + "underload": pixels with this count should be excluded, + "unit_cell": unit cell dimensions in Angstrom/degree { + "a": , "b": , "c": , + "alpha": , "beta": , "gamma": + }, +} +``` +`user_data` is defined as `header_appendix` in the `/start` operation in the `jfjoch_broker`. +Other metadata are also carried over from `/start` operation. + +If the `header_appendix` is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. +For example `header_appendix` of `{"param1": "test1", "param2": ["test1", "test2"]}`, than example message will look as follows: +```json +{ + "filename": "dataset_name_data_000001.h5", + "nimages": 1000, + "file_number": 0, + "sample_name": "my_sample", + "run_name": "my_run", + "run_number": 25, + "experiment_group": "p00001", + "beam_x_pxl": 1200, + "beam_y_pxl": 1500, + "detector_distance_m": 0.155, + "detector_height_pxl": 2164, + "detector_width_pxl": 2068, + "image_time_s": 0.001, + "nimages": 2, + "incident_energy_eV": 12400.0, + "pixel_size_m": 7.5e-05, + "saturation": 32766, + "space_group_number": 96, + "underload": -32768, + "unit_cell": { + "a": 78.0, + "alpha": 90.0, + "b": 78.0, + "beta": 90.0, + "c": 39.0, + "gamma": 90.0 + }, + "user_data": { + "param1": "test1", + "param2": ["test1", "test2"] + } +} +``` + +Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled. + +## HDF5 file structure + +Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, +integration, azimuthal integration, per-image statistics and timing) stored *beyond* the NXmx +standard. The complete file layout — master vs data files, the three format variants +(`NXmxLegacy`, `NXmxVDS`, `NXmxIntegrated`), every NXmx field that is populated and every +Jungfraujoch extension — is documented in [HDF5 / NeXus data format](HDF5.md). + +If data collection was configured with a `header_appendix` containing a key `hdf5` whose value is a +JSON object of numbers and strings, those entries are written to `/entry/user`. + +## Other formats (CBF and TIFF) +Earlier versions could also write Crystallographic Binary File (CBF, miniCBF) and TIFF images. These +writers have been removed: Jungfraujoch now writes only NXmx HDF5. The `CBF` and `TIFF` values are +retained in the file-format enum for wire back-compatibility, but a request to write either format +is rejected. + +## No file option(s) +There are two options to disable writing of files by the writer: +* Setting `file_prefix` to empty string - this will disable sending files on ZeroMQ image socket. +* Setting file format to `NoFile` - files are streamed over ZeroMQ socket, but `jfjoch_writer` will not write anything. +This can be useful for debugging purposes, or if you only rely on republishing functionality of the `jfjoch_writer` \ No newline at end of file diff --git a/_sources/LICENSE.md.txt b/_sources/LICENSE.md.txt new file mode 100644 index 00000000..222b84ba --- /dev/null +++ b/_sources/LICENSE.md.txt @@ -0,0 +1,690 @@ +# License + +Jungfraujoch software is licensed with GPLv3 license. +Jungfraujoch FPGA is licensed with CERN OHL-S license (see [FPGA license](FPGA_LICENSE.md)). + +## GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +### Preamble + +The GNU General Public License is a free, copyleft license for +software and other kinds of works. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + +Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + +Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and +modification follow. + +### TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +1. Source Code. + +The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + +The Corresponding Source for a work in source code form is that +same work. + +2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + +4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + +a) The work must carry prominent notices stating that you modified +it, and giving a relevant date. + +b) The work must carry prominent notices stating that it is +released under this License and any conditions added under section +7. This requirement modifies the requirement in section 4 to +"keep intact all notices". + +c) You must license the entire work, as a whole, under this +License to anyone who comes into possession of a copy. This +License will therefore apply, along with any applicable section 7 +additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no +permission to license the work in any other way, but it does not +invalidate such permission if you have separately received it. + +d) If the work has interactive user interfaces, each must display +Appropriate Legal Notices; however, if the Program has interactive +interfaces that do not display Appropriate Legal Notices, your +work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + +a) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by the +Corresponding Source fixed on a durable physical medium +customarily used for software interchange. + +b) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by a +written offer, valid for at least three years and valid for as +long as you offer spare parts or customer support for that product +model, to give anyone who possesses the object code either (1) a +copy of the Corresponding Source for all the software in the +product that is covered by this License, on a durable physical +medium customarily used for software interchange, for a price no +more than your reasonable cost of physically performing this +conveying of source, or (2) access to copy the +Corresponding Source from a network server at no charge. + +c) Convey individual copies of the object code with a copy of the +written offer to provide the Corresponding Source. This +alternative is allowed only occasionally and noncommercially, and +only if you received the object code with such an offer, in accord +with subsection 6b. + +d) Convey the object code by offering access from a designated +place (gratis or for a charge), and offer equivalent access to the +Corresponding Source in the same way through the same place at no +further charge. You need not require recipients to copy the +Corresponding Source along with the object code. If the place to +copy the object code is a network server, the Corresponding Source +may be on a different server (operated by you or a third party) +that supports equivalent copying facilities, provided you maintain +clear directions next to the object code saying where to find the +Corresponding Source. Regardless of what server hosts the +Corresponding Source, you remain obligated to ensure that it is +available for as long as needed to satisfy these requirements. + +e) Convey the object code using peer-to-peer transmission, provided +you inform other peers where the object code and Corresponding +Source of the work are being offered to the general public at no +charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + +a) Disclaiming warranty or limiting liability differently from the +terms of sections 15 and 16 of this License; or + +b) Requiring preservation of specified reasonable legal notices or +author attributions in that material or in the Appropriate Legal +Notices displayed by works containing it; or + +c) Prohibiting misrepresentation of the origin of that material, or +requiring that modified versions of such material be marked in +reasonable ways as different from the original version; or + +d) Limiting the use for publicity purposes of names of licensors or +authors of the material; or + +e) Declining to grant rights under trademark law for use of some +trade names, trademarks, or service marks; or + +f) Requiring indemnification of licensors and authors of that +material by anyone who conveys the material (or modified versions of +it) with contractual assumptions of liability to the recipient, for +any liability that these contractual assumptions directly impose on +those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + +13. Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +### How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + +You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + +The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + + +## Jungfraujoch exceptions to GPL + +As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL. + +We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose +of simulating FPGA design on CPU. + +If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch +API it may be distributed under terms of your choosing without being subject to GPL requirements. diff --git a/_sources/NAMING.md.txt b/_sources/NAMING.md.txt new file mode 100644 index 00000000..fa6a7a83 --- /dev/null +++ b/_sources/NAMING.md.txt @@ -0,0 +1,60 @@ +# Naming + +The software is Swiss, and so are its names: both halves of the system are named after +places in the Alps that are, in one way or another, about moving a *lot* of something up a +steep mountain as efficiently as possible — usually by train. Throughput, in other words. + +| Part | Name | What it does | +| --- | --- | --- | +| Streaming / acquisition | **Jungfraujoch** | Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing. | +| Data processing | **Rugnux** | Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the [`rugnux`](RUGNUX.md) tool). | + +## Jungfraujoch + +The **Jungfraujoch** is a high mountain col in the Bernese Alps, the saddle (*Joch* is German +for "yoke" or "col") between the peaks **Jungfrau** and **Mönch**, at 3,466 m. It is the site of +the [High Altitude Research Station Jungfraujoch](https://www.hfsjg.ch/), whose long-running +atmospheric measurements are **co-operated by the Paul Scherrer Institute** — the same institute +that develops this software and the JUNGFRAU detector. + +The name is also a small piece of word-play. PSI's **JUNGFRAU** detector and DECTRIS's **EIGER** +detector are both named after Bernese Alps peaks (the famous trio is *Eiger*, *Mönch*, *Jungfrau*). +The Jungfraujoch — the pass *between* Jungfrau and Mönch — is where those two detector worlds meet. + +And it fits the theme of the whole project: the Jungfraujoch is reached by the **Jungfraubahn**, +whose terminus is the **highest railway station in Europe** (3,454 m, the "Top of Europe"). It is +the closest you can get to that summit in a genuinely *high-throughput* way — by train, moving +crowds up the mountain — which is exactly what the streaming side of this software does with +detector frames. + +**Pronunciation (German):** *Jungfraujoch* ≈ **YUNG-frow-yokh**. +"Jung" as in *young*, "frau" rhymes with *cow*, and the final "joch" ends in the guttural *ch* of +Scottish *loch* or German *Bach* — not a hard *k*. + +## Rugnux + +**Piz Rugnux** is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern +Switzerland. (*Piz* is the Romansh word for "peak".) It rises above the **Albula line** of the +**Rhaetian Railway** (*Rhätische Bahn*), part of the "Rhaetian Railway in the Albula / Bernina +Landscapes" — a **UNESCO World Heritage Site** (*Welterbe*). + +That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude +in very little horizontal distance, it corkscrews through a series of **helical (spiral) tunnels** +looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a +steep mountain efficiently — the same idea the data-processing side of this software is built +around: pushing a large volume of diffraction data through the analysis pipeline. + +So the theme is consistent — **Swiss mountains, trains, and throughput** — while keeping the two +subsystems clearly distinct: *Jungfraujoch* streams, *Rugnux* processes. + +**Pronunciation (Romansh):** *Piz Rugnux* ≈ **peets roo-NYOOKS**. +The "gn" is a soft palatal *ñ*, as in *canyon* or Italian *gnocchi*, not two separate sounds. + +## What is Romansh? + +**Romansh** (*Rumantsch*) is the **fourth national language of Switzerland**, alongside German, +French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind +in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the +canton of Graubünden. It survives in several regional idioms, brought together in a standard form +called *Rumantsch Grischun*. Naming the processing engine with a Romansh mountain is a small nod to +the least-spoken but no-less-Swiss corner of the country. diff --git a/_sources/OPENAPI.md.txt b/_sources/OPENAPI.md.txt new file mode 100644 index 00000000..a13a3280 --- /dev/null +++ b/_sources/OPENAPI.md.txt @@ -0,0 +1,13 @@ +# OpenAPI +## OpenAPI specs + +See document with detailed [OpenAPI specs](OPENAPI_SPECS.rst). + +## Python client +Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. +For convenience, we provide Python client as [jfjoch-client](https://pypi.org/project/jfjoch-client/) PyPi package. +To install the client you can use `pip` tool: +``` +pip install jfjoch-client +``` +See [API reference from the OpenAPI generator](python_client/README.md). \ No newline at end of file diff --git a/_sources/OPENAPI_SPECS.rst.txt b/_sources/OPENAPI_SPECS.rst.txt new file mode 100644 index 00000000..15bf4e7f --- /dev/null +++ b/_sources/OPENAPI_SPECS.rst.txt @@ -0,0 +1,4 @@ +OpenAPI specification +===================== + +See document with detailed `OpenAPI specs <_static/redoc-static.html>`_ generated with Redocly. \ No newline at end of file diff --git a/_sources/PIXEL_MASK.md.txt b/_sources/PIXEL_MASK.md.txt new file mode 100644 index 00000000..033809ca --- /dev/null +++ b/_sources/PIXEL_MASK.md.txt @@ -0,0 +1,51 @@ +# Pixel mask + +## Mask format + +Jungfraujoch follows generally [NXmx format](https://manual.nexusformat.org/classes/applications/NXmx.html) format for pixel mask. +Pixel mask is described as 32-bit unsigned integer array of size the same as the image. +Conditions to mask pixel are described by setting a particular bit to one. This way it is possible to encode reason why pixel is included in the pixel mask, also for one pixel there can be multiple reasons encoded at the same time. + +Bit values are set as follows: + +Bit 0 - gap (pixel with no sensor) + +Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn't set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask) + +Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization) + +Bit 8 - user defined mask + +Bit 9 - beam stop shadow (found by `rugnux --detect-beam-stop`, on by default; see [rugnux](RUGNUX.md)). +Unlike the other bits this one belongs to the run that found it, not to the dataset: rugnux clears it +at the start of every run, so a mask read back from a file that carries one starts clear. The user +mask (bit 8) is left alone. + +Bit 30 - module edge (only for PSI systems) + +Bit 31 - chip edge interpolated pixel (multipixel) + +## Custom user mask + +Jungfraujoch allows to upload custom user mask. This happens in two steps. First create mask in TIFF format: + +```python +import numpy as np +import tifffile as tiff + +# Create a 2068x2164 numpy array filled with zeros, with 32-bit unsigned integers +array = np.zeros((2068, 2164), dtype=np.uint32) + +# Mark the pixel (300, 400) with the value 1 +array[300, 400] = 1 + +# Save the array as a TIFF file +tiff.imwrite('mask.tiff', array) +``` + +Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8). + +Then upload the mask to Jungfraujoch server: +```shell +curl -v http:///config/user_mask.tiff -XPUT --data-binary @mask.tiff +``` diff --git a/_sources/RELEASE_CONTENTS.md.txt b/_sources/RELEASE_CONTENTS.md.txt new file mode 100644 index 00000000..5b131f8f --- /dev/null +++ b/_sources/RELEASE_CONTENTS.md.txt @@ -0,0 +1,115 @@ +# Release contents + +This page describes **what a Jungfraujoch release ships and what each artefact needs on the target +machine** — which CPU instruction set the binaries were compiled for, which CUDA toolkit they were +built against, and which runtime libraries are bundled rather than expected from the host. + +The artefacts in the table below are built and published by the continuous-integration pipeline +(`.gitea/workflows/build_and_test.yml`) when a tag is pushed. For *how* to install and configure the +result see [Deployment](DEPLOYMENT.md); for the package-repository URLs see +[Linux package repositories](REPOSITORIES.md). + +## Artefacts + +| Artefact | Distributed via | Contains | +| --- | --- | --- | +| `.rpm` / `.deb` packages | [package repositories](REPOSITORIES.md) | The full server stack: `jfjoch` (broker, frontend, FPGA and detector tools), `jfjoch-writer`, `jfjoch-viewer` (incl. the XDS plugin), `jfjoch-driver-dkms` | +| `jfjoch_viewer--linux-cuda.tgz`, `...-linux-cpu.tgz` | Gitea release page | Portable Linux viewer package: `jfjoch_viewer`, `rugnux`, `jfjoch_extract_hkl`, `jfjoch_recompress` and the license notices | +| `jfjoch-viewer--win64-cuda.exe`, `...-win64-cpu.exe` | Gitea release page | Windows installer with the same four programs, plus the Qt runtime | +| `jfjoch-writer` `.rpm` / `.deb` | Gitea release page | The writer alone, for a file-writing machine without the rest of the stack | +| `libjfjoch_xds_plugin.so.` | Gitea release page | XDS HDF5 read plugin (built on RHEL 8); see [Integration with MX software](SOFTWARE_INTEGRATION.md) | +| `jfjoch-client` | [PyPI](https://pypi.org/project/jfjoch-client/) and the Gitea PyPI index | Generated Python OpenAPI client | +| Documentation | [Read the Docs](https://jungfraujoch.readthedocs.io) and the `gitea-pages` branch | This documentation set | + +The FPGA firmware (`.mcs`) images are attached to the release as well. The firmware is stable and is +carried from version to version, and is rebuilt with Vivado (see [FPGA smartNIC](FPGA.md)) when it +needs to change — so a card keeps its image across a software upgrade unless the release notes say +otherwise. + +## CPU instruction set + +The architecture flags live in the CI configuration rather than in `CMakeLists.txt`, so a site +building from source picks its own (`x86-64-v4` on an AVX-512 cluster, `-march=native`, or the plain +baseline the compiler defaults to). The released binaries are compiled to a fixed floor: + +| Release | Flags | Minimum CPU | +| --- | --- | --- | +| Linux (all packages, and the portable `.tgz`) | `-march=x86-64-v3 -flto=auto` | AVX2 + FMA + BMI2 — Intel Haswell (2013) / AMD Zen (2017) and newer | +| Windows installer | `/arch:AVX` | AVX — Intel Sandy Bridge (2011) / AMD Bulldozer and newer | + +The Windows floor is lower because MSVC has no spelling for the `x86-64-v2` level; `/arch:AVX` is +the nearest one and implies SSE4.1/4.2, which is what actually matters — without it Eigen has no +vectorised `round` and falls back to a libm call per element. Link-time optimisation is applied on +Linux only. + +A binary will fault with an illegal instruction on a CPU below its floor. If you must run on older +hardware, build from source without the flags. + +## Operating-system floor + +The `.rpm` / `.deb` packages are built per distribution (RHEL/Rocky 8 and 9, Ubuntu 22.04 and 24.04) +and are tied to it. The portable viewer `.tgz` is built on RHEL 8, the oldest supported +distribution, so its glibc floor is low enough to run on any newer Linux — that is what it is for, +and why it replaces the per-distro packaging of the viewer on the release page. The Windows +installer is built and verified on Windows 11. + +## CUDA and non-CUDA builds + +Every binary artefact is released in **two variants**, `cuda` and `cpu`. The CUDA variant adds +the GPU fast-feedback indexer (`ffbidx`), the GPU FFT indexer and GPU image processing; the CPU-only +variant runs the same pipeline on the CPU with the FFTW indexer, at much lower throughput. + +The CUDA toolkit used is the one on the corresponding build machine: **CUDA 12** for the RHEL 8 +packages, **CUDA 13** for RHEL 9, Ubuntu and Windows. The major version is part of the artefact and +repository name, so a download is self-identifying. Building from source needs CUDA 12.8 or newer. + +**A CUDA build does not require a CUDA machine.** Of the CUDA components only **cuFFT** is linked +dynamically — the CUDA runtime and the fast-feedback indexer are linked statically — and cuFFT +itself has no link-time dependency on the NVIDIA driver library. Jungfraujoch asks how many CUDA +devices are present at start-up and treats "none" (including "no driver installed") as zero GPUs, +falling back to the CPU path. So a CUDA build starts and runs correctly on a machine with no NVIDIA +GPU at all, provided the cuFFT runtime can be loaded: + +- **Portable `.tgz` and Windows installer** — cuFFT is **part of the distribution**, shipped next to + the executable (on Linux found through an `$ORIGIN` rpath). Nothing else is needed: no CUDA + toolkit, and on a GPU machine only the NVIDIA driver. +- **`.rpm` / `.deb`** — cuFFT comes from the distribution's own CUDA packages, so that one + dependency is managed centrally with the rest of CUDA. Install the cuFFT package alongside, or use + the `nocuda` repositories on a machine where CUDA is not wanted. + +The cuFFT runtime is large (the Windows DLL is ~256 MB), so the CUDA artefacts are correspondingly +bigger than the CPU ones — the other reason for shipping both. + +On a machine with an NVIDIA GPU, take the CUDA variant: only that one uses the GPU. + +## Windows installer + +The Windows artefact covers `jfjoch_viewer` and the portable analysis CLIs only; the rest of +Jungfraujoch (broker, receiver, FPGA host, detector control) is Linux-only. + +The toolchain bounds of the released installer are: + +- **Visual Studio 2026** with the C++ (MSVC) toolset. MSVC is not optional — CUDA on Windows builds + through it — and it is what the release is compiled with. +- **CUDA Toolkit 13.3** for the `cuda13` variant. +- **Qt 6.11** for MSVC (`msvc2022_64`), including Qt Charts. +- Ninja as the generator; zlib and Eigen 3.4 supplied from a build prefix. + +The installer is generated with NSIS and **bundles the Qt runtime** (via `windeployqt`) and, on the +CUDA variant, the cuFFT DLL — so the end user installs neither Qt nor a CUDA toolkit. The two +variants share an install directory and Start Menu group and replace each other (CUDA is a strict +superset); they are told apart by the installer filename and the Add/Remove Programs entry: + +| Build | Installer file | Add/Remove Programs | +| --- | --- | --- | +| CUDA (default) | `jfjoch-viewer--win64-cuda.exe` | `Jungfraujoch (CUDA)` | +| CPU-only | `jfjoch-viewer--win64-cpu.exe` | `Jungfraujoch (CPU)` | + +To build the viewer yourself on Windows, see +[jfjoch_viewer ▸ Building from source on Windows](JFJOCH_VIEWER.md#building-from-source-on-windows). + +## Licenses + +Every package variant carries the project license, the third-party manifest and the verbatim +license texts of the bundled dependencies under `share/doc/jfjoch`. See +[Third-party software notices](THIRD_PARTY_NOTICES.md). diff --git a/_sources/REPOSITORIES.md.txt b/_sources/REPOSITORIES.md.txt new file mode 100644 index 00000000..133f8b6f --- /dev/null +++ b/_sources/REPOSITORIES.md.txt @@ -0,0 +1,53 @@ +# Linux package repositories +For convenience, we are providing package repositories. With versions including and excluding CUDA linking. +We recommend to install Jungfraujoch viewer from `nocuda` repository and remaining packages from `cuda12`/`cuda13` repository. + +The repository name encodes two choices: the [slsDetectorPackage](DETECTORS.md) version the packages +were built against (`slsdet8` = 8.0.2, `slsdet9` = 9.2.0 — it must match the detector firmware) and +whether CUDA is linked in. What ends up inside each package, and what it needs on the target +machine, is described in [Release contents](RELEASE_CONTENTS.md). + +## RHEL based systems + +For RHEL systems we provide the following repositories: + +| RHEL version | slsDetectorPackage | CUDA | Repository file | +|--------------|--------------------|------|------------------------------------------------------------------------| +| 8.x | 8.0.2 | 12.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo | +| 8.x | 9.2.0 | 12.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12.repo | +| 8.x | 8.0.2 | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo | +| 9.x | 8.0.2 | 13.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13.repo | +| 9.x | 9.2.0 | 13.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13.repo | +| 9.x | 8.0.2 | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda.repo | + +To install the repository, run: + +```bash +dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo +``` +RPMs are signed by the Gitea package registry as they are uploaded. If your system cannot verify the +signature, set `gpgcheck=0` in the repository file or install with `--nogpgcheck`. + +We provide the following packages in the repository: +* jfjoch — broker, web frontend, FPGA and detector command-line tools +* jfjoch-driver-dkms — PCIe kernel-module source, built by DKMS +* jfjoch-writer — HDF5 writer service +* jfjoch-viewer — desktop viewer, the offline analysis tools and the XDS plugin + +## Ubuntu based systems + +For Ubuntu systems, we also provide the following repositories: +``` +sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc +echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list +sudo apt update +``` + +`$distribution` uses Ubuntu names `jammy` (22.04) and `noble` (24.04). `$component` can be set to `cuda13` and `nocuda`. +Only slsDetectorPackage 8.0.2 is built for Ubuntu. + +The same four packages as above are provided: `jfjoch`, `jfjoch-driver-dkms`, `jfjoch-writer` and +`jfjoch-viewer`. Up to 1.0.0-rc.160 the first of them was misnamed `jfjoch-jfjoch`; the current +package replaces it, so `apt upgrade` handles the rename. + +Ubuntu packages are currently only going through a very limited testing. diff --git a/_sources/RUGNUX.md.txt b/_sources/RUGNUX.md.txt new file mode 100644 index 00000000..c26598a1 --- /dev/null +++ b/_sources/RUGNUX.md.txt @@ -0,0 +1,510 @@ +# rugnux + +`rugnux` is the **offline** crystallographic data-analysis tool of Jungfraujoch — the +data-processing half of the system (see [Naming](NAMING.md) for where the name comes from). +It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, +geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the +results to a `_process.h5` file, plus reflection files (`.mtz`/`.cif`/`.hkl`) when merging is +requested. + +It runs the *same* analysis code as the online and interactive tools, just driven from the +command line over a file rather than a live detector stream. + +> **Note.** `rugnux` is under very active development. This page describes the tool and +> its options at a high level; the authoritative, always-current list of options is the program's +> own usage message — run `rugnux` with no arguments. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| [`jfjoch_viewer`](JFJOCH_VIEWER.md) | Interactive, on-screen exploration | Qt desktop application | On screen; a processing job can write the same files as `rugnux` | +| **`rugnux`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | + +Use `rugnux` to re-analyse data after acquisition, to experiment with processing +parameters, or to produce merged intensities for downstream structure solution. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. The CUDA build +provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`fft`); without CUDA +only the CPU `fftw` indexer is available. Spot finding, integration and scaling run on the CPU and +scale with the thread count (`-N`). + +## Input and output + +**Input** is a single Jungfraujoch HDF5 master file (NXmx-based). Spots are always found by `rugnux` +itself, including for the two-pass rotation first pass — the spot lists a dataset may already carry +were found online, at the acquisition's threshold and with its ice-band spots already discarded, so +reusing them would hide the spot-finding settings from the lattice search. + +**Output** (controlled by `-o, --output-prefix`, default `output`): + +- `_process.h5` — NXmx-compliant HDF5 with derived metadata (spots, indexing, + integration, azimuthal integration, per-image statistics). See + [HDF5 / NeXus data format](HDF5.md) for the layout. Written by default only when **not** merging + (i.e. under `--no-merge`); add `--write-process-h5` to also write it when merging. +- Merging is **on by default** (`--no-merge` disables it). The merged reflections are written in + **three** formats — each has its uses downstream: + - `.mtz` — CCP4 MTZ (`IMEAN`/`I(+)`/`I(-)`, French–Wilson `F`, `FreeR_flag`) for the CCP4 / + phenix reflection tools. + - `.cif` — mmCIF, for deposition and as the self-describing native format (also carries the + merging statistics, ISa, twinning and radiation-damage indicators). + - `.hkl` — SHELX **HKLF 4** text (`h k l I σ(I)`, fixed `3I4,2F8.2`), the direct input for + **SHELXC / ANODE / SHELXD**. Bijvoet mates are written separately (`I(+)` at `+hkl`, `I(-)` at + `-hkl`) so the anomalous signal is preserved; intensities are put on a common scale so the largest + value fits the fixed-width field (the absolute scale is irrelevant to SHELXC/ANODE), and the file + ends with the `0 0 0` terminator record. + + All three carry the **refined unit cell** (from rotation indexing) and the **space group determined + from systematic absences** (constrained to the indexed lattice symmetry). No-reference scaling + additionally emits per-iteration `_iterN_scale.dat`. +- `_report.txt` — the **results report**: what the run determined, in a form both a person and + a beamline script can read. Always written, next to the files above. See + [The results report](#the-results-report) below. + +Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the +console. By default the written resolution is trimmed automatically where CC1/2 falls off +(`--resolution-cutoff cc-logistic`, CC1/2 target 0.30); set `--scaling-high-resolution` to fix the +limit by hand, or `--resolution-cutoff off` to keep the full range. + +### Reflection-file conventions + +**mmCIF.** Standard items carry their standard meanings — `_refln.intensity_meas` / `_intensity_sigma`, +the `pdbx_I_plus`/`pdbx_I_minus` and `pdbx_F_plus`/`pdbx_F_minus` anomalous pairs, `_reflns.*` and +`_reflns_shell.*` for the merging statistics, `_reflns.B_iso_Wilson_estimate` for the Wilson B, and +`_cell.*` / `_diffrn_radiation_wavelength.wavelength` for the geometry. + +Anything rugnux reports that has no standard item is written under a **`jfjoch_`** prefix, inside the +standard category it belongs to. That is a deliberate choice: a reader that does not know these items +ignores them, and one that does can find them without guessing. + +| item | meaning | +|---|---| +| `_reflns.jfjoch_diffrn_ISa` | Asymptotic I/σ in **XDS's sense**: the whole-range `1/√(a·b)` of the error model, so it can be read directly against a `CORRECT.LP` | +| `_reflns.jfjoch_diffrn_ISa_asymptotic` | The **strong-reflection** tier — the counting-subtracted scatter of well-measured groups. XDS has no equivalent, and it can only ever be the more optimistic of the two. Rotation path only | +| `_reflns.jfjoch_error_model_a`, `_b` | The error model in XDS's convention, `σ² = a(σ₀² + b·I²)`, so the ISa above is re-derivable from the file rather than taken on trust | +| `_reflns.jfjoch_second_moment_I` | Twinning second moment ⟨I²⟩/⟨I⟩² — 2.00 untwinned, 1.50 for a perfect twin | +| `_reflns.jfjoch_L_test_mean_abs_L`, `_L_test_mean_L_squared` | Padilla–Yeates L-test. ⟨\|L\|⟩ is 0.500 untwinned / 0.375 for a perfect twin; ⟨L²⟩ is 0.333 / 0.200. Written only when the test found pairs | +| `_reflns.jfjoch_radiation_damage_relative_B` | Relative *B* from the first to the last rotation batch (Ų); positive is the usual direction, high-resolution intensity fading with dose | +| `_jfjoch_radiation_damage_batch.*` | Per-batch loop: `id`, `rotation_start_deg`, `relative_B` | +| `_diffrn_detector.jfjoch_distance_mm`, `_jfjoch_beam_center_x_pxl`, `_jfjoch_beam_center_y_pxl` | The refined detector geometry actually used, which is not otherwise recoverable from the reflection file | + +> **Compatibility note.** Before rc.161, `_reflns.jfjoch_diffrn_ISa` carried the *asymptote*, not the +> whole-range value. There is no version marker inside the file, so a number taken from an older +> `.cif` is not comparable with one taken from a newer one. + +**SHELX HKLF 4** (`.hkl`). Fixed-format `3I4,2F8.2` — `h k l I σ(I)`, one record per +reflection, terminated by a `0 0 0` record — which is what **SHELXC**, **SHELXD** and **ANODE** +expect. Two properties worth knowing before using it: + +- **Bijvoet mates are written separately**, `I(+)` at `+hkl` and `I(-)` at `-hkl`, so the anomalous + differences survive into SHELXC; a reflection with no anomalous split is written once, as its mean. +- **Intensities are rescaled** by a single global factor so the largest value fits the `F8.2` field. + `I` and `σ(I)` share that factor, so every ratio — and therefore the anomalous signal — is + untouched, but the absolute scale is not meaningful. This matters only if you intend to compare + magnitudes with another file; SHELXC and ANODE use ratios alone. + +## The results report + +`_report.txt` records **what the run determined**, next to the reflection files. It is +written on every `--mode mx` and `--mode scale` run that has an output prefix — there is no option +to enable or disable it. Two cases follow from that: + +- An **empty output prefix** (`-o ""`, the "compute the statistics, persist nothing" mode) writes + nothing, the report included. +- **`--no-merge`** still writes a report. It determined an indexing and a geometry result, and those + are recorded; the merging section then says `MERGE= NOT_PERFORMED` rather than being omitted, so + the absence is a statement and not something a reader has to infer. + +The report is never allowed to fail a run: if it cannot be written (unwritable path, full disk) the +failure is logged as a warning and the run finishes normally. + +### Format + +The model is XDS's `CORRECT.LP`: prose and tables a crystallographer reads top to bottom, with a +structure a script can consume without parsing prose. + +- **`KEY= value` assignment lines.** Every number worth extracting is one, so a consumer gets it with + a single `grep '^ISA= '` and never has to read a sentence. Key names are stable. +- **Fixed-width tables** with a stable header row for anything that is genuinely tabular — the + resolution shells, the space-group candidates, the sweep-quality ranges. +- **`WARNING:` lines**, one per finding, in plain English: `WARNING: Frames 500-600 out of beam + (10.1 deg, scale 0.12 and CC 0.30 of the run, 2% scaled)`. `grep '^WARNING:'` finds every one. +- **Section banners** (`***…***` around a numbered title) delimiting the blocks. + +`REPORT_VERSION=` is the format's own version. Key names, table columns and the reason vocabulary +below are an interface other software may depend on: they do not change without that number moving. + +Sections, in order: `1. DATA SET`, `2. INDEXING`, `3. GEOMETRY POST-REFINEMENT` (rotation only), +`4. SPACE GROUP DETERMINATION`, `5. SCALING AND MERGING`, `6. TWINNING`, `7. RADIATION DAMAGE`, +`8. SWEEP QUALITY`, `9. WARNINGS`. + +**Which pass.** A rotation run integrates twice — once at the geometry in the input file +(`_01.*`), then again at the post-refined geometry (`.*`) — and can integrate a +third time if a guard rejects the second pass. There is **one** report, for the pass that became the +canonical output, and `PASS=` / `PASS_DECISION=` in section 1 say which pass that is and on what +evidence, so no number in the file is ambiguous about which geometry produced it. + +**Not in the report:** timing, frame rates, thread counts, per-image progress and library banners. +Those are process, not result, and stay on stdout. + +### Sweep quality and the reason vocabulary + +Section 8 lists the stretches of the sweep over which the crystal delivered much less than the rest +of the run — the feedback a beamline control system needs to tell an operator that a crystal should +be recentred or recollected. Nothing is excluded on the strength of it; the frames still carry +signal, and this is a message for the beamline, not a filter. + +``` +SWEEP_QUALITY_STATUS= COMPUTED +SWEEP_QUALITY_COUNT= 1 +SWEEP_QUALITY_REASONS= no_diffraction crystal_out_of_beam weak_diffraction loss_of_centring radiation_damage +SWEEP_ROTATION= 360.0 +FLUX_PEAK_TO_TROUGH= 1.03 +SCALE_MODULATION_PEAK_TO_TROUGH= 1.00 + + FIRST_IMAGE LAST_IMAGE N_IMAGES ROTATION REASON SEVERITY SCALE CC INDEXED + ----------- ----------- --------- -------- -------------------- -------- ------ ------ -------- + 500 600 101 10.1 crystal_out_of_beam 0.83 0.12 0.30 0.02 + ----------- ----------- --------- -------- -------------------- -------- ------ ------ -------- +``` + +`SWEEP_QUALITY_STATUS` distinguishes **`COMPUTED`** (the diagnostic ran; a count of 0 means the sweep +was clean throughout) from **`NOT_COMPUTED`** (it did not run — no scaling and merging, or stills +data). A consumer must not read a missing table or a zero count as "clean" without checking it. +`SWEEP_QUALITY_REASONS` lists the whole vocabulary this version can emit, so an unknown code is +distinguishable from a missing one. + +| Reason code | Meaning | +|-------------|---------| +| `no_diffraction` | The range recorded essentially no diffraction from the indexed lattice. | +| `crystal_out_of_beam` | Frames were lost: over the range a per-image scale could be fitted far less often than over the run. | +| `weak_diffraction` | The frames all still index, but with much less intensity — the cause was not determined. | +| `loss_of_centring` | One cycle of modulation per revolution: the crystal is off the rotation axis. | +| `radiation_damage` | The range runs to the end of a sweep whose quality was already decaying. | + +The vocabulary is **closed and stable**: a code is never renamed, and never reused for a different +meaning. New codes are only ever added, and adding one moves `REPORT_VERSION`. + +The columns are: `FIRST_IMAGE`/`LAST_IMAGE` — inclusive, in processed-image ordinals (the numbering +of `_image.dat` and of every other per-image array `rugnux` writes; with `-s`/`--stride` the +source image is `start + ordinal * stride`); `ROTATION` — the width of the range in degrees; +`SEVERITY` — the fraction of the run's typical diffracting power missing over the range, 0 (as good +as the run) to 1 (nothing at all); `SCALE` and `CC` — the range's mean per-image scale and +CC-to-merge relative to the run median; `INDEXED` — the fraction of the range's frames that were +scaled at all. Every range also appears as a `WARNING:` sentence in section 9. + +The same finding is written **per image** into the `_process.h5` as `/entry/MX/sweepQuality`, when +one is written — see [HDF5](HDF5.md#41-entrymx--spot-finding-and-indexing-cxi-style). + +## Validating against a model (`rugnux --model`) + +Given a PDB atomic model of the same structure, `--model model.pdb` scales the model structure +factors to the merged amplitudes — fitting a flat bulk-solvent contribution and an overall +anisotropic *B* — and reports **R-work / R-free** and the mean 2Fo-Fc density at the atom centres. +It also writes `_2fofc.ccp4`, `_fofc.ccp4` and `_maps.mtz` next to the +merged reflections. Nothing about the model is refined; it is only re-fractionalized into the data +cell, so a deposited model with a slightly different cell still lines up. + +It is a *data-quality lens*, independent of the internal statistics: R-free measures the merged +intensities against external truth, where CC1/2 and Rmeas only measure them against +themselves. It also settles the two things merged intensities alone cannot: the enantiomorph (data +merged in P41212 against a P43212 model are reindexed +into the model's hand), and — when no reference MTZ has already fixed it — a merohedral indexing +ambiguity, by keeping the candidate reindexing with the lowest R-free. + +## Re-scaling and re-merging (`rugnux --mode scale`) + +The `scale` mode re-scales and merges the *already-integrated* reflections stored in a +`_process.h5` file, without re-running spot finding or integration. Use it to re-merge quickly with a +different space group, resolution limit, anomalous setting or reference MTZ. It reuses the same +`-o/-N/-s/-e/-S/-A/-B/-z/--scaling-*` options as the full run, and (unlike the full pipeline) does +not run a space-group search, so pass `-S` for the correct symmetry. + +## Detector calibration from powder rings (`rugnux --mode calibration`) + +The `calibration` mode determines the detector geometry — PONI *x*/*y*, the two tilts +rot1/rot2 and the distance — from the powder rings of a calibrant, and writes it as a +pyFAI **`.poni`** file alongside a printed report of how far each parameter moved from +the header. Bragg data pin the beam centre worst (it is gauge-coupled to the crystal orientation); +a powder ring has no orientation to be coupled to, so this is the measurement that fixes it. + +``` +rugnux --mode calibration --calibrant lab6 -N 8 -o det LaB6_master.h5 +``` + +`--calibrant` takes `lab6`, `agbh` (silver behenate), `ceo2`, `si` or `ice`, case-insensitively. +**`ice`** calibrates a real experiment against its own ice rings — no calibrant exposure needed — +and is the reason a calibrant is a list of ring positions rather than a unit cell: hexagonal ice +is P63/mmc, so rings enumerated from its cell would include systematically absent ones. + +`--calibration` picks how the rings are measured, and **both use the whole dataset** — `-s`/`-e`/`-t` +select which images: + +- **`rings`** (default) sums the (*q* × azimuth) azimuthal profile over every processed image into + one map and fits the ring arcs in it. A powder ring is an arc, not a set of spots, and the summed + profile measures it at every azimuth with all the run's counts behind it. It needs the profile to + be binned in azimuth, so this mode defaults `--azim-phi-bins` to 32. +- **`spots`** pools the found spots of every processed image and fits those. It determines the + centre from scratch (a Hough circle vote, which quantises it to a whole pixel) and then refines. + +Both routes read the ring position out of a binned profile or a spot centroid, so the radial +sampling matters: at a long detector distance the default 0.01 Å⁻¹ *q* bin is several pixels wide +and quantises the `rings` route accordingly — pass a finer `--azim-q-spacing` there (the total +*q* × azimuth bin count must stay under 65534). + +The report prints the fitted geometry, the scatter of the ring points about the fitted rings and the +standard error that implies on the centre. That error is *formal*: it measures the scatter of the +points, not whether the rings themselves are trustworthy, so it stays small when a fit goes wrong +for a structural reason — one visible ring, or ice that is textured rather than smooth. + +Both the PONI (the point of normal incidence, which is what a `.poni` file stores) and the direct +beam (where the beam lands, which is what most other programs call the beam centre) are printed. +They differ by *distance* × tan(rot) once the detector is tilted, which on a 0.3° tilt at 300 mm is +several pixels — enough to look like a disagreement with another program when there is none. + +## Quick start + +### Rotation data + +Index, integrate, scale and merge a rotation sweep, fully de novo: + +``` +rugnux rotation_master.h5 \ + -o rotation_run -N 32 \ + --scaling-high-resolution 1.4 +``` + +Because the dataset carries a rotation goniometer axis, it is processed as **rotation data by +default**: two-pass rotation indexing (index the sweep once, then process every frame against that +lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). Scaling +and merging run **by default** (for both rotation and stills; `--no-merge` turns them off); the unit +cell is taken from the rotation indexer and the space group is determined from systematic absences, +and both are written +into the merged `.cif`. + +Run **fully de novo** (no `-C`/`-S`) for the best result — supplying a cell or space group up front +tends to *degrade* low-symmetry cases. `--scaling-high-resolution` (set it to your expected +resolution) sharpens both the space-group search and the error model. To tune the first pass use +`--two-pass-rotation=100` (or `-R100` — the first-pass image count); to force the sweep to be +treated as independent stills use `--force-still`. + +By default a rotation run also **post-refines the geometry** in a second pass: the first pass +integrates and merges at the header geometry, then the detector distance + beam centre and the crystal +cell / rotation-axis are refined against the merged fulls (cross-validated, and committed only for a +small < 1 % move, with the gauge-weak beam centre restrained toward the header), and the second pass +re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical +`_*` output; the header-geometry pass is kept alongside as `_01_*` for comparison. +Disable it with `--rotation-no-postrefine`. + +After the per-frame scale-fulls step, rotation scaling applies three **correction surfaces**, **on by +default** (`--no-scaling-corrections` disables all): + +- **Decay** — a global Debye–Waller relative-*B* over the run, for the radiation damage that weakens + later frames more at high resolution (a resolution×time systematic the resolution-flat per-frame + scale cannot remove). It only engages when the total relative-*B* exceeds a physical floor (2 Ų). An + optional `--relative-b[=deg]` extends this single global rate to a smooth per-batch relative-*B* curve + (default 10°-of-rotation batches when bare, off otherwise), cross-validated like the surfaces here, for + crystals whose decay is non-linear in dose. +- **Absorption** — a smooth multiplicative factor over the diffracted-beam direction in the goniometer + frame (path length through the crystal). Negligible at hard X-rays / thin crystals; it matters at + low photon energy. Its benefit shows up most on model-based metrics: a smooth absorption error + largely cancels among symmetry mates (little effect on the error model / ISa) but still biases the + intensities, so it measurably lowers *R*free. +- **Modulation** — a smooth multiplicative factor over the position where a reflection lands on the + detector (a flat-field: detector-response and geometric systematics that vary across the detector + plane). Symmetry-equivalents of one reflection land at different detector positions as the crystal + rotates, which over-determines the surface. Because it lives in the detector frame (not the + rotation) the same correction concept applies to stills. This is the largest of the three on + JUNGFRAU data — it lowers *R*meas by several to tens of percent on datasets that carry a + detector systematic, while holding or improving CC1/2 and the anomalous signal. + +All three are **cross-validated** — fitted on even-numbered frames and kept only if they improve the +held-out odd-frame symmetry-equivalent agreement by a clear margin (and vice versa). The agreement is +scored as a σ-independent, *R*meas-like fractional deviation, so a surface can never pass +cross-validation by merely reshaping the sigmas; where the systematic is absent the surface is a no-op +rather than a source of added noise, which is why they are safe to leave on. + +Independently of any correction, a rotation run prints a **radiation-damage report** — the per-image +scale correlation-to-merge and mosaicity versus dose, and the relative *B*-factor change over the run +(first→last) together with a per-batch relative-*B* curve, also written to the merged mmCIF. It is a +data-quality-vs-dose diagnostic and never alters the merged intensities. A batch whose data cannot +support a measurement prints `-` instead of a value, and the first→last number is printed only where a +straight line describes the curve — damage is progressive, so a curve that dips and recovers is a +disturbance of the sweep, not dose, and the report says so and points at the sweep-quality section +(`RADIATION_DAMAGE_RELATIVE_B= NOT_A_TREND`). + +### Still / serial data + +A dataset with **no goniometer axis** (e.g. a serial grid scan) is processed as **independent +stills automatically** — no flag needed. Known-cell indexing with the GPU fast-feedback indexer, +then merge against a reference structure: + +``` +rugnux serial_master.h5 \ + -o serial_run -N 32 \ + -X ffbidx -C 79,79,38,90,90,90 -S 96 \ + -z reference.mtz \ + --scaling-high-resolution 1.8 +``` + +`ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills. The +self-calibrating spot finder is on by default for both workflows (`--no-adaptive-spots` turns it off), and for +serial stills leave `--min-pix-per-spot` **unset** so it is chosen per image — across the still-target battery this +combination raises the indexing rate and typically extends resolution over a fixed threshold and +fixed min-pix, at equal or better CC½. (You can still pin a fixed threshold with `--spot-sigma` / +`--spot-threshold` and a fixed min-pix with `--min-pix-per-spot`.) If a dataset *does* carry a +goniometer axis but you want per-frame stills processing anyway, add `--force-still`. + +## Command-line options + +General: + +| Option | Description | +| --- | --- | +| `-o, --output-prefix ` | Output file prefix (default: `output`) | +| `-N, --threads ` | Number of worker threads (default: all hardware threads) | +| `-s, --start-image ` | First image to process (default: 0) | +| `-e, --end-image ` | Last image to process (default: all) | +| `-t, --stride ` | Process every *n*-th image (default: 1) | +| `-v, --verbose` | Verbose output | + +Mode — `--mode ` (default `mx`): + +| Value | Description | +| --- | --- | +| `mx` | Full analysis — spot finding, indexing, integration and merging | +| `azint` | Only azimuthal integration (no spot finding/indexing); writes `_process.h5` | +| `scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) | +| `calibration` | Determine the detector geometry from powder rings; writes `.poni` | + +Calibration (`--mode calibration`): + +| Option | Description | +| --- | --- | +| `--calibrant ` | Powder standard: `lab6` \| `agbh` \| `ceo2` \| `si` \| `ice` (default `lab6`, case-insensitive) | +| `--calibration ` | How the rings are measured: `rings` \| `spots` (default `rings`; see above). `rings` defaults `--azim-phi-bins` to 32 | + +Detector mask: + +| Option | Description | +| --- | --- | +| `--detect-beam-stop[=N\|off]` | Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. **On by default** (60 images); `=off` disables. Reflections behind the stop are attenuated but not flagged, so they integrate low with a plausible sigma and no existing rejection catches them | + +Geometry: + +| Option | Description | +| --- | --- | +| `--estimate-beam-center` | Measure the direct beam before indexing, from the symmetry of the spots where the sweep reaches at least half a turn and from the radial background profile where it does not; the value in the file is kept where neither can measure it. Off by default | +| `--no-fit-spindle` | With the above, keep the rotation axis given in the file instead of fitting its skew about the beam | + +Spot finding: + +| Option | Description | +| --- | --- | +| `--spot-sigma ` | Noise sigma level for spot finding (default: 4.0) | +| `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | +| `--adaptive-spots` | Self-calibrating detection (**default**, stills and rotation alike): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed `--spot-threshold`, so one setting adapts across datasets (no per-dataset `--spot-threshold`/`--spot-sigma` tuning) | +| `--no-adaptive-spots` | Turn adaptive detection off and use the fixed `--spot-threshold` / `--spot-sigma` finder | +| `--spot-false-pixels ` | Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies `--adaptive-spots`) | +| `--spot-high-resolution ` | High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills | +| `--spot-low-resolution ` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit) | +| `--min-pix-per-spot ` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. | +| `--max-spots ` | Maximum spots kept per image (the strongest ones) and handed to indexing (default: 1000) | +| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** | + +Azimuthal integration (the radial profile behind the per-image ice-ring score): + +| Option | Description | +| --- | --- | +| `-q, --azim-q-spacing ` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) | +| `--azim-min-q ` | Minimum Q, 1/Å | +| `--azim-max-q ` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see | +| `--azim-phi-bins ` | Number of azimuthal (phi) bins (default: 1) | +| `--polarization-correction ` | Enable/disable the azimuthal polarization correction | +| `--solid-angle-correction ` | Enable/disable the azimuthal solid-angle correction | + +Indexing: + +A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation +indexing) by default; a dataset without one is processed as independent stills. `--force-still` +overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request +rotation explicitly and pick the pass or lattice. + +| Option | Description | +| --- | --- | +| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | +| `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`) | +| `-S, --space-group ` | Space group number (`92`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling | +| `-r, --refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) | +| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) | +| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) | +| `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass | +| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see the rotation section) | +| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), `=off` disables | +| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) | + +Indexer choice in brief: `ffbidx` (GPU) refines toward a **known cell** and is best for sparse +serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the +[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md) for the algorithms. + +Scaling and merging: + +| Option | Description | +| --- | --- | +| `--no-merge` | Skip scaling and merging (on by default); write only the per-image `_process.h5` | +| `-A, --anomalous` | Anomalous mode (keep Friedel pairs separate) | +| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | +| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) | +| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below) | +| `--relative-b[=deg]` | rot3d: fit a per-batch relative-*B* beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise) | +| `--simple-stills` | Stills: treat every reflection as a full (*p* = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement | +| `--no-expected-variance-merge` | Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting | +| `--capture-uncertainty ` | rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise) | +| `--min-captured-fraction ` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) | +| `--scaling-high-resolution ` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) | +| `--scaling-low-resolution ` | Low-resolution limit for scaling and merging, Å (default: 50, the value XDS configurations use; 0 removes the limit). Reflections coarser than this sit behind or beside the beam stop and are measured on a background it has eaten into | +| `--resolution-cutoff ` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) | +| `--resolution-cc-target ` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) | +| `--resolution-shells ` | Number of resolution shells in the reported statistics table (default: 10) | +| `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02) | +| `--ice-min-score ` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The eleven fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness | +| `--ice-min-spot-ratio ` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) | +| `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for `rot3d`, off otherwise) | +| `--min-image-cc ` | Per-image CC limit, percent (default: no limit) | +| `--search-min-zeta ` | De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry \|ζ\| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences | +| `--mosaicity ` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed | +| `--scaling-iterations ` | Scaling iterations with no reference data (default: 3) | +| `-z, --reference-mtz ` | Reference MTZ (enables reference-driven scaling) | +| `--reference-column