commit 4f45e3d8c9fc3bd02488d9128b9224b74f682fa3 Author: Filip Leonarski (Gitea) Date: Sun Jul 12 19:03:50 2026 +0000 Deploy site diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 00000000..b2f5b7f4 --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 79c22211b6d0f9eb5db189cef418880c +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/ACKNOWLEDGEMENT.doctree b/.doctrees/ACKNOWLEDGEMENT.doctree new file mode 100644 index 00000000..a8a0a52d Binary files /dev/null and b/.doctrees/ACKNOWLEDGEMENT.doctree differ diff --git a/.doctrees/CBOR.doctree b/.doctrees/CBOR.doctree new file mode 100644 index 00000000..cd4506b0 Binary files /dev/null and b/.doctrees/CBOR.doctree differ diff --git a/.doctrees/CHANGELOG.doctree b/.doctrees/CHANGELOG.doctree new file mode 100644 index 00000000..1219a28f Binary files /dev/null and b/.doctrees/CHANGELOG.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS.doctree b/.doctrees/CPU_DATA_ANALYSIS.doctree new file mode 100644 index 00000000..000124ab Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/DEPLOYMENT.doctree b/.doctrees/DEPLOYMENT.doctree new file mode 100644 index 00000000..497358fa Binary files /dev/null and b/.doctrees/DEPLOYMENT.doctree differ diff --git a/.doctrees/DETECTORS.doctree b/.doctrees/DETECTORS.doctree new file mode 100644 index 00000000..1741b8a1 Binary files /dev/null and b/.doctrees/DETECTORS.doctree differ diff --git a/.doctrees/DETECTOR_GEOMETRY.doctree b/.doctrees/DETECTOR_GEOMETRY.doctree new file mode 100644 index 00000000..07e2f641 Binary files /dev/null and b/.doctrees/DETECTOR_GEOMETRY.doctree differ diff --git a/.doctrees/FPGA.doctree b/.doctrees/FPGA.doctree new file mode 100644 index 00000000..150b650f Binary files /dev/null and b/.doctrees/FPGA.doctree differ diff --git a/.doctrees/FPGA_DATA_ANALYSIS.doctree b/.doctrees/FPGA_DATA_ANALYSIS.doctree new file mode 100644 index 00000000..f9ac7872 Binary files /dev/null and b/.doctrees/FPGA_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/FPGA_DESIGN.doctree b/.doctrees/FPGA_DESIGN.doctree new file mode 100644 index 00000000..eb7b37e3 Binary files /dev/null and b/.doctrees/FPGA_DESIGN.doctree differ diff --git a/.doctrees/FPGA_LICENSE.doctree b/.doctrees/FPGA_LICENSE.doctree new file mode 100644 index 00000000..9e76d8d4 Binary files /dev/null and b/.doctrees/FPGA_LICENSE.doctree differ diff --git a/.doctrees/FPGA_NETWORK.doctree b/.doctrees/FPGA_NETWORK.doctree new file mode 100644 index 00000000..4fc62c88 Binary files /dev/null and b/.doctrees/FPGA_NETWORK.doctree differ diff --git a/.doctrees/FPGA_PCIE_DRIVER.doctree b/.doctrees/FPGA_PCIE_DRIVER.doctree new file mode 100644 index 00000000..dd8427f5 Binary files /dev/null and b/.doctrees/FPGA_PCIE_DRIVER.doctree differ diff --git a/.doctrees/FPGA_SETTINGS.doctree b/.doctrees/FPGA_SETTINGS.doctree new file mode 100644 index 00000000..f69e27f9 Binary files /dev/null and b/.doctrees/FPGA_SETTINGS.doctree differ diff --git a/.doctrees/HARDWARE.doctree b/.doctrees/HARDWARE.doctree new file mode 100644 index 00000000..e1d2766b Binary files /dev/null and b/.doctrees/HARDWARE.doctree differ diff --git a/.doctrees/HDF5.doctree b/.doctrees/HDF5.doctree new file mode 100644 index 00000000..b2ff7e08 Binary files /dev/null and b/.doctrees/HDF5.doctree differ diff --git a/.doctrees/IMAGE_STREAM.doctree b/.doctrees/IMAGE_STREAM.doctree new file mode 100644 index 00000000..9d46a2bb Binary files /dev/null and b/.doctrees/IMAGE_STREAM.doctree differ diff --git a/.doctrees/JFJOCH_BROKER.doctree b/.doctrees/JFJOCH_BROKER.doctree new file mode 100644 index 00000000..1bd9c5a9 Binary files /dev/null and b/.doctrees/JFJOCH_BROKER.doctree differ diff --git a/.doctrees/JFJOCH_VIEWER.doctree b/.doctrees/JFJOCH_VIEWER.doctree new file mode 100644 index 00000000..a1673123 Binary files /dev/null and b/.doctrees/JFJOCH_VIEWER.doctree differ diff --git a/.doctrees/JFJOCH_WRITER.doctree b/.doctrees/JFJOCH_WRITER.doctree new file mode 100644 index 00000000..88e98f65 Binary files /dev/null and b/.doctrees/JFJOCH_WRITER.doctree differ diff --git a/.doctrees/LICENSE.doctree b/.doctrees/LICENSE.doctree new file mode 100644 index 00000000..728fbb6b Binary files /dev/null and b/.doctrees/LICENSE.doctree differ diff --git a/.doctrees/NAMING.doctree b/.doctrees/NAMING.doctree new file mode 100644 index 00000000..7badc096 Binary files /dev/null and b/.doctrees/NAMING.doctree differ diff --git a/.doctrees/OPENAPI.doctree b/.doctrees/OPENAPI.doctree new file mode 100644 index 00000000..0bce0219 Binary files /dev/null and b/.doctrees/OPENAPI.doctree differ diff --git a/.doctrees/OPENAPI_SPECS.doctree b/.doctrees/OPENAPI_SPECS.doctree new file mode 100644 index 00000000..2c353cb8 Binary files /dev/null and b/.doctrees/OPENAPI_SPECS.doctree differ diff --git a/.doctrees/PIXEL_MASK.doctree b/.doctrees/PIXEL_MASK.doctree new file mode 100644 index 00000000..667281bd Binary files /dev/null and b/.doctrees/PIXEL_MASK.doctree differ diff --git a/.doctrees/REPOSITORIES.doctree b/.doctrees/REPOSITORIES.doctree new file mode 100644 index 00000000..b8b722b9 Binary files /dev/null and b/.doctrees/REPOSITORIES.doctree differ diff --git a/.doctrees/RUGNUX.doctree b/.doctrees/RUGNUX.doctree new file mode 100644 index 00000000..a36c2f01 Binary files /dev/null and b/.doctrees/RUGNUX.doctree differ diff --git a/.doctrees/SOFTWARE.doctree b/.doctrees/SOFTWARE.doctree new file mode 100644 index 00000000..c3ccfed1 Binary files /dev/null and b/.doctrees/SOFTWARE.doctree differ diff --git a/.doctrees/SOFTWARE_INTEGRATION.doctree b/.doctrees/SOFTWARE_INTEGRATION.doctree new file mode 100644 index 00000000..9efde916 Binary files /dev/null and b/.doctrees/SOFTWARE_INTEGRATION.doctree differ diff --git a/.doctrees/TESTS.doctree b/.doctrees/TESTS.doctree new file mode 100644 index 00000000..d39c369b Binary files /dev/null and b/.doctrees/TESTS.doctree differ diff --git a/.doctrees/THIRD_PARTY_NOTICES.doctree b/.doctrees/THIRD_PARTY_NOTICES.doctree new file mode 100644 index 00000000..32fb3b07 Binary files /dev/null and b/.doctrees/THIRD_PARTY_NOTICES.doctree differ diff --git a/.doctrees/TOOLS.doctree b/.doctrees/TOOLS.doctree new file mode 100644 index 00000000..df94aae3 Binary files /dev/null and b/.doctrees/TOOLS.doctree differ diff --git a/.doctrees/VERSIONING.doctree b/.doctrees/VERSIONING.doctree new file mode 100644 index 00000000..326131b3 Binary files /dev/null and b/.doctrees/VERSIONING.doctree differ diff --git a/.doctrees/WEB_FRONTEND.doctree b/.doctrees/WEB_FRONTEND.doctree new file mode 100644 index 00000000..c3cefca9 Binary files /dev/null and b/.doctrees/WEB_FRONTEND.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 00000000..23c1dd15 Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 00000000..8cbdbf05 Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/python_client/README.doctree b/.doctrees/python_client/README.doctree new file mode 100644 index 00000000..b9c18719 Binary files /dev/null and b/.doctrees/python_client/README.doctree differ diff --git a/.doctrees/python_client/docs/AzimIntSettings.doctree b/.doctrees/python_client/docs/AzimIntSettings.doctree new file mode 100644 index 00000000..587014b8 Binary files /dev/null and b/.doctrees/python_client/docs/AzimIntSettings.doctree differ diff --git a/.doctrees/python_client/docs/BrokerStatus.doctree b/.doctrees/python_client/docs/BrokerStatus.doctree new file mode 100644 index 00000000..132fe9db Binary files /dev/null and b/.doctrees/python_client/docs/BrokerStatus.doctree differ diff --git a/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree new file mode 100644 index 00000000..2f30d3f5 Binary files /dev/null and b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree differ diff --git a/.doctrees/python_client/docs/ColorScale.doctree b/.doctrees/python_client/docs/ColorScale.doctree new file mode 100644 index 00000000..db282e1f Binary files /dev/null and b/.doctrees/python_client/docs/ColorScale.doctree differ diff --git a/.doctrees/python_client/docs/DarkMaskSettings.doctree b/.doctrees/python_client/docs/DarkMaskSettings.doctree new file mode 100644 index 00000000..b7ed23aa Binary files /dev/null and b/.doctrees/python_client/docs/DarkMaskSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettings.doctree b/.doctrees/python_client/docs/DatasetSettings.doctree new file mode 100644 index 00000000..6974dd71 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree new file mode 100644 index 00000000..ef560a43 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsUnitCell.doctree b/.doctrees/python_client/docs/DatasetSettingsUnitCell.doctree new file mode 100644 index 00000000..3571ff9a Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsUnitCell.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree new file mode 100644 index 00000000..1b34cc7b Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree differ diff --git a/.doctrees/python_client/docs/DefaultApi.doctree b/.doctrees/python_client/docs/DefaultApi.doctree new file mode 100644 index 00000000..e49f1ff5 Binary files /dev/null and b/.doctrees/python_client/docs/DefaultApi.doctree differ diff --git a/.doctrees/python_client/docs/Detector.doctree b/.doctrees/python_client/docs/Detector.doctree new file mode 100644 index 00000000..006089ef Binary files /dev/null and b/.doctrees/python_client/docs/Detector.doctree differ diff --git a/.doctrees/python_client/docs/DetectorList.doctree b/.doctrees/python_client/docs/DetectorList.doctree new file mode 100644 index 00000000..f951b0d7 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorList.doctree differ diff --git a/.doctrees/python_client/docs/DetectorListDetectorsInner.doctree b/.doctrees/python_client/docs/DetectorListDetectorsInner.doctree new file mode 100644 index 00000000..29b80735 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorListDetectorsInner.doctree differ diff --git a/.doctrees/python_client/docs/DetectorListElement.doctree b/.doctrees/python_client/docs/DetectorListElement.doctree new file mode 100644 index 00000000..dda31eb6 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorListElement.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModule.doctree b/.doctrees/python_client/docs/DetectorModule.doctree new file mode 100644 index 00000000..bfdcac79 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModule.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModuleDirection.doctree b/.doctrees/python_client/docs/DetectorModuleDirection.doctree new file mode 100644 index 00000000..b510009f Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModuleDirection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorPowerState.doctree b/.doctrees/python_client/docs/DetectorPowerState.doctree new file mode 100644 index 00000000..115ce332 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorPowerState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSelection.doctree b/.doctrees/python_client/docs/DetectorSelection.doctree new file mode 100644 index 00000000..765b84ce Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSelection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSettings.doctree b/.doctrees/python_client/docs/DetectorSettings.doctree new file mode 100644 index 00000000..2557ede2 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSettings.doctree differ diff --git a/.doctrees/python_client/docs/DetectorState.doctree b/.doctrees/python_client/docs/DetectorState.doctree new file mode 100644 index 00000000..84750d04 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorStatus.doctree b/.doctrees/python_client/docs/DetectorStatus.doctree new file mode 100644 index 00000000..316cb6cb Binary files /dev/null and b/.doctrees/python_client/docs/DetectorStatus.doctree differ diff --git a/.doctrees/python_client/docs/DetectorTiming.doctree b/.doctrees/python_client/docs/DetectorTiming.doctree new file mode 100644 index 00000000..22dc0c03 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorTiming.doctree differ diff --git a/.doctrees/python_client/docs/DetectorType.doctree b/.doctrees/python_client/docs/DetectorType.doctree new file mode 100644 index 00000000..f4e0a6a9 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorType.doctree differ diff --git a/.doctrees/python_client/docs/ErrorMessage.doctree b/.doctrees/python_client/docs/ErrorMessage.doctree new file mode 100644 index 00000000..41404f38 Binary files /dev/null and b/.doctrees/python_client/docs/ErrorMessage.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterFormat.doctree b/.doctrees/python_client/docs/FileWriterFormat.doctree new file mode 100644 index 00000000..4641379d Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterFormat.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterSettings.doctree b/.doctrees/python_client/docs/FileWriterSettings.doctree new file mode 100644 index 00000000..5ece661c Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterSettings.doctree differ diff --git a/.doctrees/python_client/docs/FpgaStatusInner.doctree b/.doctrees/python_client/docs/FpgaStatusInner.doctree new file mode 100644 index 00000000..ed1965ab Binary files /dev/null and b/.doctrees/python_client/docs/FpgaStatusInner.doctree differ diff --git a/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree new file mode 100644 index 00000000..6814aa9c Binary files /dev/null and b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/GridPlot.doctree b/.doctrees/python_client/docs/GridPlot.doctree new file mode 100644 index 00000000..97ea6123 Binary files /dev/null and b/.doctrees/python_client/docs/GridPlot.doctree differ diff --git a/.doctrees/python_client/docs/GridPlots.doctree b/.doctrees/python_client/docs/GridPlots.doctree new file mode 100644 index 00000000..60b7fd22 Binary files /dev/null and b/.doctrees/python_client/docs/GridPlots.doctree differ diff --git a/.doctrees/python_client/docs/GridScan.doctree b/.doctrees/python_client/docs/GridScan.doctree new file mode 100644 index 00000000..7552bf22 Binary files /dev/null and b/.doctrees/python_client/docs/GridScan.doctree differ diff --git a/.doctrees/python_client/docs/GridScanResult.doctree b/.doctrees/python_client/docs/GridScanResult.doctree new file mode 100644 index 00000000..cafaec9f Binary files /dev/null and b/.doctrees/python_client/docs/GridScanResult.doctree differ diff --git a/.doctrees/python_client/docs/GridScanResultImagesInner.doctree b/.doctrees/python_client/docs/GridScanResultImagesInner.doctree new file mode 100644 index 00000000..196925c0 Binary files /dev/null and b/.doctrees/python_client/docs/GridScanResultImagesInner.doctree differ diff --git a/.doctrees/python_client/docs/ImageBufferStatus.doctree b/.doctrees/python_client/docs/ImageBufferStatus.doctree new file mode 100644 index 00000000..04efdce5 Binary files /dev/null and b/.doctrees/python_client/docs/ImageBufferStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImageFormatSettings.doctree b/.doctrees/python_client/docs/ImageFormatSettings.doctree new file mode 100644 index 00000000..d795ce4b Binary files /dev/null and b/.doctrees/python_client/docs/ImageFormatSettings.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherStatus.doctree b/.doctrees/python_client/docs/ImagePusherStatus.doctree new file mode 100644 index 00000000..2be4badd Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherType.doctree b/.doctrees/python_client/docs/ImagePusherType.doctree new file mode 100644 index 00000000..b9e53dc3 Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherType.doctree differ diff --git a/.doctrees/python_client/docs/IndexingAlgorithm.doctree b/.doctrees/python_client/docs/IndexingAlgorithm.doctree new file mode 100644 index 00000000..e150689f Binary files /dev/null and b/.doctrees/python_client/docs/IndexingAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/IndexingSettings.doctree b/.doctrees/python_client/docs/IndexingSettings.doctree new file mode 100644 index 00000000..fe809b57 Binary files /dev/null and b/.doctrees/python_client/docs/IndexingSettings.doctree differ diff --git a/.doctrees/python_client/docs/InstrumentMetadata.doctree b/.doctrees/python_client/docs/InstrumentMetadata.doctree new file mode 100644 index 00000000..769d4b73 Binary files /dev/null and b/.doctrees/python_client/docs/InstrumentMetadata.doctree differ diff --git a/.doctrees/python_client/docs/JfjochBrokerApi.doctree b/.doctrees/python_client/docs/JfjochBrokerApi.doctree new file mode 100644 index 00000000..1d7b0175 Binary files /dev/null and b/.doctrees/python_client/docs/JfjochBrokerApi.doctree differ diff --git a/.doctrees/python_client/docs/JfjochSettings.doctree b/.doctrees/python_client/docs/JfjochSettings.doctree new file mode 100644 index 00000000..4f8597fe Binary files /dev/null and b/.doctrees/python_client/docs/JfjochSettings.doctree differ diff --git a/.doctrees/python_client/docs/JfjochSettingsSsl.doctree b/.doctrees/python_client/docs/JfjochSettingsSsl.doctree new file mode 100644 index 00000000..f1f9aa25 Binary files /dev/null and b/.doctrees/python_client/docs/JfjochSettingsSsl.doctree differ diff --git a/.doctrees/python_client/docs/JfjochStatistics.doctree b/.doctrees/python_client/docs/JfjochStatistics.doctree new file mode 100644 index 00000000..10ad5f87 Binary files /dev/null and b/.doctrees/python_client/docs/JfjochStatistics.doctree differ diff --git a/.doctrees/python_client/docs/MeasurementStatistics.doctree b/.doctrees/python_client/docs/MeasurementStatistics.doctree new file mode 100644 index 00000000..a5e56977 Binary files /dev/null and b/.doctrees/python_client/docs/MeasurementStatistics.doctree differ diff --git a/.doctrees/python_client/docs/PcieDevicesInner.doctree b/.doctrees/python_client/docs/PcieDevicesInner.doctree new file mode 100644 index 00000000..909a5bfd Binary files /dev/null and b/.doctrees/python_client/docs/PcieDevicesInner.doctree differ diff --git a/.doctrees/python_client/docs/PixelMaskStatistics.doctree b/.doctrees/python_client/docs/PixelMaskStatistics.doctree new file mode 100644 index 00000000..d3e9f09d Binary files /dev/null and b/.doctrees/python_client/docs/PixelMaskStatistics.doctree differ diff --git a/.doctrees/python_client/docs/Plot.doctree b/.doctrees/python_client/docs/Plot.doctree new file mode 100644 index 00000000..b9bb9578 Binary files /dev/null and b/.doctrees/python_client/docs/Plot.doctree differ diff --git a/.doctrees/python_client/docs/PlotTypeEnum.doctree b/.doctrees/python_client/docs/PlotTypeEnum.doctree new file mode 100644 index 00000000..437ebc96 Binary files /dev/null and b/.doctrees/python_client/docs/PlotTypeEnum.doctree differ diff --git a/.doctrees/python_client/docs/PlotUnitX.doctree b/.doctrees/python_client/docs/PlotUnitX.doctree new file mode 100644 index 00000000..5f0ace53 Binary files /dev/null and b/.doctrees/python_client/docs/PlotUnitX.doctree differ diff --git a/.doctrees/python_client/docs/Plots.doctree b/.doctrees/python_client/docs/Plots.doctree new file mode 100644 index 00000000..299ab240 Binary files /dev/null and b/.doctrees/python_client/docs/Plots.doctree differ diff --git a/.doctrees/python_client/docs/PreviewSettings.doctree b/.doctrees/python_client/docs/PreviewSettings.doctree new file mode 100644 index 00000000..9024be2c Binary files /dev/null and b/.doctrees/python_client/docs/PreviewSettings.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimList.doctree b/.doctrees/python_client/docs/RoiAzimList.doctree new file mode 100644 index 00000000..f9a57907 Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimList.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimuthal.doctree b/.doctrees/python_client/docs/RoiAzimuthal.doctree new file mode 100644 index 00000000..9f8e9084 Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimuthal.doctree differ diff --git a/.doctrees/python_client/docs/RoiBox.doctree b/.doctrees/python_client/docs/RoiBox.doctree new file mode 100644 index 00000000..929be63d Binary files /dev/null and b/.doctrees/python_client/docs/RoiBox.doctree differ diff --git a/.doctrees/python_client/docs/RoiBoxList.doctree b/.doctrees/python_client/docs/RoiBoxList.doctree new file mode 100644 index 00000000..65cdc1ef Binary files /dev/null and b/.doctrees/python_client/docs/RoiBoxList.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircle.doctree b/.doctrees/python_client/docs/RoiCircle.doctree new file mode 100644 index 00000000..3fd5dacf Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircle.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircleList.doctree b/.doctrees/python_client/docs/RoiCircleList.doctree new file mode 100644 index 00000000..ea9ddcaf Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircleList.doctree differ diff --git a/.doctrees/python_client/docs/RoiDefinitions.doctree b/.doctrees/python_client/docs/RoiDefinitions.doctree new file mode 100644 index 00000000..e282ae2c Binary files /dev/null and b/.doctrees/python_client/docs/RoiDefinitions.doctree differ diff --git a/.doctrees/python_client/docs/RotationAxis.doctree b/.doctrees/python_client/docs/RotationAxis.doctree new file mode 100644 index 00000000..17b00e70 Binary files /dev/null and b/.doctrees/python_client/docs/RotationAxis.doctree differ diff --git a/.doctrees/python_client/docs/ScanResult.doctree b/.doctrees/python_client/docs/ScanResult.doctree new file mode 100644 index 00000000..c6b20533 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResult.doctree differ diff --git a/.doctrees/python_client/docs/ScanResultImagesInner.doctree b/.doctrees/python_client/docs/ScanResultImagesInner.doctree new file mode 100644 index 00000000..d6e83392 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResultImagesInner.doctree differ diff --git a/.doctrees/python_client/docs/SpotFindingSettings.doctree b/.doctrees/python_client/docs/SpotFindingSettings.doctree new file mode 100644 index 00000000..c05a19e8 Binary files /dev/null and b/.doctrees/python_client/docs/SpotFindingSettings.doctree differ diff --git a/.doctrees/python_client/docs/StandardDetectorGeometry.doctree b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree new file mode 100644 index 00000000..04e06fea Binary files /dev/null and b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree differ diff --git a/.doctrees/python_client/docs/TcpSettings.doctree b/.doctrees/python_client/docs/TcpSettings.doctree new file mode 100644 index 00000000..355545e8 Binary files /dev/null and b/.doctrees/python_client/docs/TcpSettings.doctree differ diff --git a/.doctrees/python_client/docs/UnitCell.doctree b/.doctrees/python_client/docs/UnitCell.doctree new file mode 100644 index 00000000..536658f2 Binary files /dev/null and b/.doctrees/python_client/docs/UnitCell.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree new file mode 100644 index 00000000..6af290dc Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree new file mode 100644 index 00000000..cbd200c2 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqSettings.doctree b/.doctrees/python_client/docs/ZeromqSettings.doctree new file mode 100644 index 00000000..633af6f5 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqSettings.doctree differ diff --git a/ACKNOWLEDGEMENT.html b/ACKNOWLEDGEMENT.html new file mode 100644 index 00000000..b7d3ca70 --- /dev/null +++ b/ACKNOWLEDGEMENT.html @@ -0,0 +1 @@ + Acknowledgements — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Acknowledgements

Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang “Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors” (2023), J. Synchrotron Rad., 30, 227-234 doi:10.1107/S1600577522010268.

The project is supported by :

  • Innosuisse via Innovation Project “NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications” (101.535.1 IP-ENG; Apr 2023 - Sep 2025).

  • ETH Domain via Open Research Data Contribute project (Jan - Dec 2023)

  • AMD University Program with donation of licenses of Ethernet IP cores and Vivado software

This software uses Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license

\ No newline at end of file diff --git a/CBOR.html b/CBOR.html new file mode 100644 index 00000000..b0f6b852 --- /dev/null +++ b/CBOR.html @@ -0,0 +1 @@ + CBOR messages — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

CBOR messages

To communicate between FPGA-equipped receiver system and writers, Jungfraujoch is using binary CBOR encoding with tinycbor library (Intel). The protocol is based on and compatible with DECTRIS Stream2. There are minor differences at the moment:

  • LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed

  • Few fields are currently absent

  • Extra fields are present beyond DECTRIS standard

  • There are calibration and metadata messages defined beyond DECTRIS specification

Start message

Field name

Type

Description

Present in DECTRIS format

type

String

value “start”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

detector_distance

float

Detector distance [m]

detector_translation

Array(float)

Detector translation vector [m]

X

beam_center_x

float

Beam center in X direction [pixels]

X

beam_center_y

float

Beam center in Y direction [pixels]

X

countrate_correction_enabled

bool

Countrate correction enabled

X

flatfield_enabled

bool

Flatfield enabled

X

number_of_images

uint64

Number of images in the series

X

image_size_x

uint64

Image width [pixels]

X

image_size_y

uint64

Image height [pixels]

X

incident_energy

float

X-ray energy [eV]

X

incident_wavelength

float

X-ray wavelength [Angstrom]

X

incident_wavelength_spread

float (optional)

FWHM of the X-ray wavelength distribution [Angstrom] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic

frame_time

float

Frame time, if multiple frames per trigger [s]

X

count_time

float

Exposure time [s]

X

saturation_value

int64

Maximum valid sample value

X

error_value

int64 (optional)

Value used in images to describe pixels that are in error state or missing

pixel_size_x

float

Pixel width [m]

X

pixel_size_y

float

Pixel height [m]

X

sensor_thickness

float

Sensor thickness [m]

X

sensor_material

string

Sensor material

X

arm_date

date

Approximate date of arming

X

pixel_mask_enabled

bool

Pixel mask applied on images

X

detector_description

string

Name of the detector

X

detector_serial_number

string

Detector serial number

X

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

fluorescence

object (optional)

X-ray fluorescence spectrum collected at start

- energy

Array(float)

Energy of measuring point [eV]

- data

Array(float)

Fluorescence scan result data [arbitrary units]; must be strictly the same length as energy

goniometer

Map

Definition of rotation axis (optional)

X

- AXIS

string

Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch

X

- - increment

float

Rotation axis increment (per image) in degree [deg]

X

- - start

float

Rotation axis start angle [deg]

X

- - axis

Array(float)

Vector for the rotation axis

- - helical_step

Array(float)

Translation for helical scan for 1 image [m]

- - screening_wedge

Array(float)

Wedge for screening [deg] (increment would correspond to difference between screening points)

grid_scan

object

Grid scan definition (optional and exclusive with rotation axis)

- n_fast

uint64

Number of elements along fast axis

- n_slow

uint64

Number of elements along slow axis

- step_x_axis

float

Step along X axis, can be negative [m]

- step_y_axis

float

Step along Y axis, can be negative [m]

- snake_scan

bool

Snake scan (rows alternate direction)

- vertical_scan

bool

Vertical scan (enabled: fast direction = Y, disabled: fast direction = X)

jungfrau_conversion_enabled

bool (optional)

Applying JUNGFRAU pixel conversion (to photons or keV)

jungfrau_conversion_factor

float (optional)

Factor used for JUNGFRAU conversion [eV]

geometry_transformation_enabled

bool (optional)

Transformation from detector module geometry (512x1024) to full detector geometry

pixel_mask

Map(string -> Image)

Pixel mask - multiple in case of storage cells

X

channels

Array(string)

List of image channels

X

max_spot_count

uint64

Maximum number of spots identified in spot finding

max_extra_lattices

uint64

Maximum number of extra lattices

storage_cell_number

uint64 (optional)

Number of storage cells used by JUNGFRAU

storage_cell_delay

Rational

Delay of storage cells in JUNGFRAU

threshold_energy

Map(string -> float)

Per-channel threshold energy [eV] (map of channel name to value)

image_dtype

string

Pixel bit type (e.g. uint16)

X

unit_cell

object (optional)

Unit cell of the system: a, b, c [angstrom] and alpha, beta, gamma [degree]

az_int_q_bin_count

uint64

Number of azimuthal integration bins in the radial direction

az_int_phi_bin_count

uint64

Number of azimuthal integration bins in the phi angle direction

az_int_bin_to_q

Array(float)

Q value for each azimuthal integration bin [angstrom^-1]

az_int_bin_to_two_theta

Array(float)

Two theta angle value for each azimuthal integration bin [deg]

az_int_bin_to_phi

Array(float)

Phi value for each azimuthal integration bin [deg]

az_int_map

Image

Mapping between pixel and bin number

summation

uint64

Factor of frame summation

user_data

string

JSON serialized to string that can contain the following fields (all fields are optional):

X

- file_prefix

string

File prefix

- images_per_file

uint64

Number of images written per file

- images_per_trigger

uint64

Number of images collected per trigger

- source_name

string

Facility name

- source_type

string

Type of X-ray source (use NXsource/type values, for example “Synchrotron X-ray Source” or “Free-Electron Laser”)

- instrument_name

string

Instrument name

- sample_name

string

Name of the sample

- user

any valid JSON

Value of header_appendix provided at collection start to Jungfraujoch

- attenuator_transmission

float

Attenuator transmission []

- total_flux

float

Total flux [ph/s]

- space_group_number

uint64

Space group number

- summation_mode

string

Summation mode (internal|fpga|cpu)

- overwrite

bool

Overwrite existing HDF5 files

- file_format

int

File writer format: 0 = only data files, 1 = NXmx legacy soft links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF, 5 = TIFF, 6 = no file written

- roi

Array(object)

ROI configurations; each element is one of:

type “box”: xmin, xmax, ymin, ymax (numbers)

type “circle”: r, x, y (numbers)

type “azim”: qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector

- gain_file_names

Array(string)

Names of JUNGFRAU gain files used for the current detector

- write_master_file

bool

With multiple sockets, it selects which socket will provide master file

- write_images

bool

Write images in the HDF5 file (if false, will only write metadata)

- data_reduction_factor_serialmx

uint64

Data reduction factor for serial MX

- experiment_group

string

ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number

- jfjoch_release

string

Jungfraujoch release number

- socket_number

uint64

Number of ZeroMQ socket (on jfjoch_broker side) used for transmission

- bit_depth_readout

uint64

Bit depth of the detector readout

- writer_notification_zmq_addr

string

ZeroMQ address to inform jfjoch_broker about writers that finished operation

- xfel_pulse_id

uint64

Pulse IDs are recorded for images

- ring_current_mA

float

Ring current at the start of the measurement

- sample_temperature_K

float

Sample temperature [K]

- detect_ice_rings

bool

Ice ring detection feature is enabled

- indexing_algorithm

string

Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none

- geom_refinement_algorithm

string

Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center

- poni_rot1

float

Tilt of the detector rot1 according to PyFAI PONI convention [rad]

- poni_rot2

float

Tilt of the detector rot2 according to PyFAI PONI convention [rad]

- poni_rot3

float

Tilt of the detector rot3 according to PyFAI PONI convention [rad]

See DECTRIS documentation for definition of Image as MultiDimArray with optional compression.

Image message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “image”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

real_time

Rational

Exposure time

X

start_time

Rational

Exposure start time (highly approximate)

X

end_time

Rational

Exposure end time (highly approximate)

X

spots

Array(object)

Spots:

- x

float

observed position in x (pixels)

- y

float

observed position in y (pixels)

- I

float

intensity (photons)

- maxc

int64

max count (photons)

- ice_ring

bool

spot in resolution range for ice rings

- indexed

bool

indexed solution

- latt

int64

Lattice to which the peak belongs (negative number = not indexed)

- image

int64

image number the spot belongs to

- h

int64

Miller index (indexed spots only)

- k

int64

Miller index (indexed spots only)

- l

int64

Miller index (indexed spots only)

- dist_ewald

float

distance to Ewald sphere [Angstrom^-1] (indexed spots only)

reflections

Array(object)

Reflections:

- h

int64

Miller index

- k

int64

Miller index

- l

int64

Miller index

- x

float

prediced position in x (pixels)

- y

float

predicted position in y (pixels)

- obs_x

float

observed position in x (pixels)

- obs_y

float

observed position in y (pixels)

- d

float

resolution [Angstrom]

- I

float

integrated intensity (photons)

- bkg

float

mean background value (photons)

- sigma

float

standard deviation, estimated from counting statistics (photons)

- image

float

image number (present for each spot)

- rp

float

Distance to Ewald sphere [Angstrom^-1]

- rlp

float

Reciprocal Lorentz and polarization corrections

- partiality

float

Partiality of the reflection

- phi

float

phi angle from XDS: difference from middle of current frame, not absolute [deg]

- zeta

float

Lorentz zeta factor (reciprocal-space geometry term)

- image_scale_corr

float

Per-image scale correction; I_true = image_scale_corr * I

spot_count

uint64

Spot count

spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

spot_count_low_res

uint64

Number of spots in low resolution (prior to filtering)

spot_count_indexed

uint64

Number of spots which fit indexing solution within a given tolerance

az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

NaN is used for empty bins and has to be taken care by the receiver

az_int_profile_std

Array(float)

Standard deviation for azimuthal integration. (NaN for less than 2 samples)

az_int_profile_count

Array(uint64)

Number of pixels contributing to azimuthal bin

indexing_result

bool

Indexing successful

indexing_lattice_count

int64

Number of indexing lattices found for this image

indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

indexing_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

profile_radius

float

Profile radius of the image - describes distance of observed reflections from the Ewald sphere [Angstrom^-1]

integrated_reflections

int64

Count of integrated reflections

mosaicity

float

Angular range of spots in image from a rotation scan [degree]

b_factor

float

Estimated B-factor (Angstrom^2)

compression_time

float

Time spent on compression/decompressing image [s]

preprocessing_time

float

Time spent on preparing the image for analysis [s]

azint_time

float

Time spent on azimuthal integration [s]

spot_finding_time

float

Time spent on spot finding [s]

indexing_time

float

Time spent on indexing [s]

refinement_time

float

Time spent on refinement of indexing solution and experimental geometry [s]

index_analysis_time

float

Time spent on analyzing idnexing solution, calculating profile radius and mosaicity [s]

bragg_prediction_time

float

Time spent on predicting Bragg spots [s]

integration_time

float

Time spent on Bragg integration [s]

image_scale_time

float

Time spent on on-the-fly scaling [s]

processing_time

float

Total processing time [s]

xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

lattice_type

object

Bravais lattice classification of the indexing result (present only if available)

X

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

jf_info

uint64

Detector info field

receiver_aq_dev_delay

uint64

Receiver internal delay

receiver_free_send_buf

uint64

Receiver internal number of available buffer locations

receiver_buf_in_sending

uint64

Receiver internal number of buffer locations currently in sending/writing

receiver_buf_in_preparation

uint64

Receiver internal number of buffer locations currently in processing

storage_cell

uint64

Storage cell number

saturated_pixel_count

uint64

Saturated pixel count

pixel_sum

uint64

Sum of all pixels, excl. error and saturation

error_pixel_count

uint64

Error pixel count

strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

min_viable_pixel_value

int64

Minimal pixel value, excl. error and saturation

max_viable_pixel_value

int64

Maximal pixel value, excl. error and saturation

resolution_estimate

float

Diffraction resolution estimation [Angstrom]

X

data_collection_efficiency

float

Image collection efficiency []

packets_expected

uint64

Number of packets expected per image (in units of 2 kB)

packets_received

uint64

Number of packets received per image (in units of 2 kB)

bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons]

ice_ring_score

float

Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

beam_corr_x

float

Beam center correction X applied during processing [pixel]

X

beam_corr_y

float

Beam center correction Y applied during processing [pixel]

X

image_scale_factor

float

Scaling result: Image scale factor (g)

X

image_scale_mosaicity

float

Scaling result: Image scale mosaicity [deg]

X

image_scale_b_factor

float

Scaling result: Image scale B factor [Angstrom^2]

X

image_scale_cc

float

Scaling result: Image scale CC

X

adu_histogram

Array(uint64)

ADU histogram

roi_integrals

object

Results of ROI calculation

X

- sum

int64

Sum of pixels in ROI area [photons]

- sum_square

int64

Sum of squares of pixels in ROI area [photons]

- pixels

uint64

Valid pixels in ROI area

- max_count

int64

Highest count in ROI area [photons]

- x_weighted_sum

int64

ROI pixel X position multiplied by photon count [photons * pixels]

- y_weighted_sum

int64

ROI pixel Y position multiplied by photon count [photons * pixels]

user_data

string

Optional user defined text information - this is image_appendix serialized to JSON format

X

data

Map(string -> Image)

Image

X

Metadata message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “metadata”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

images

Array(object)

Array of images (order and size of the array are not guaranteed)

X

- image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

- original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

- real_time

Rational

Exposure time

X

- start_time

Rational

Exposure start time (highly approximate)

X

- end_time

Rational

Exposure end time (highly approximate)

X

- spot_count

uint64

Spot count

- spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

- az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

- indexing_result

bool

Indexing successful

- indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

- indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

- xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

- xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

- jf_info

uint64

Detector info field

- receiver_aq_dev_delay

uint64

Receiver internal delay

- receiver_free_send_buf

uint64

Receiver internal number of available send buffers

- storage_cell

uint64

Storage cell number

- saturated_pixel_count

uint64

Saturated pixel count

- error_pixel_count

uint64

Error pixel count

- strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

- data_collection_efficiency

float

Image collection efficiency []

- bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons] (with solid angle/polarization corrections, if applied)

X

- resolution_estimate

float

Diffraction resolution estimation

X

- adu_histogram

Array(uint64)

ADU histogram

X

- roi_integrals

object

Results of ROI calculation

X

- - sum

int64

Sum of pixels in ROI area [photons]

- - sum_square

int64

Sum of squares of pixels in ROI area [photons]

- - pixels

uint64

Valid pixels in ROI area

- - max_count

int64

Highest count in ROI area [photons]

End message

Field name

Type

Description

Present in DECTRIS format

type

String

value “end”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

end_date

string

Approximate end date

max_image_number

uint64

Number of image with the highest number; counted from 1 to distinguish zero images and one image

images_collected

uint64

Number of images collected

images_sent_to_write

uint64

Number of images sent to writer; if writer queues were full, it is possible this is less than images collected

data_collection_efficiency

float

Overall network packets collected / network packets expected

az_int_result

Map(text->Array(float))

Azimuthal integration results, use az_int_bin_to_q from start message for legend

adu_histogram

Map(text->Array(uint64))

ADU values histogram

adu_histogram_bin_width

uint64

Width of bins in the above histogram [ADU]

max_receiver_delay

uint64

Internal performance of Jungfraujoch

bkg_estimate

float

Mean background estimate for the whole run

indexing_rate

float

Mean indexing rate for the whole run

unit_cell

object (optional)

Unit cell of the system, based on the actual experiment: a, b, c [angstrom] and alpha, beta, gamma [degree]

rotation_lattice_type

object

Bravais lattice classification of the total rotation solution over the run, if available; same schema as lattice_type

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

rotation_lattice

Array(9 * float)

Real-space lattice basis, flattened 3x3 in row-major order

rotation_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

data_collection_efficiency_image

Array(float)

Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context

spot_count

Array(int32)

Per-image spot count

spot_count_ice_ring

Array(int32)

Per-image number of spots within identified ice-ring resolution ranges

spot_count_low_res

Array(int32)

Per-image number of low-resolution spots

spot_count_indexed

Array(int32)

Per-image number of spots fitting indexing solution

image_indexed

Array(uint8)

Per-image indexing result; 0 = not indexed, nonzero = indexed

v_bkg_estimate

Array(float)

Per-image background estimate

ice_ring_score

Array(float)

Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice)

profile_radius

Array(float)

Per-image profile radius [Angstrom^-1]

mosaicity

Array(float)

Per-image mosaicity [degree]

bFactor

Array(float)

Per-image estimated B-factor [Angstrom^2]

resolution_estimate

Array(float)

Per-image diffraction resolution estimate [Angstrom]

min_viable_pixel_value

Array(int64)

Per-image minimum valid pixel value, excluding error/saturated pixels

max_viable_pixel_value

Array(int64)

Per-image maximum valid pixel value, excluding error/saturated pixels

saturated_pixel_count

Array(int32)

Per-image saturated pixel count

error_pixel_count

Array(int32)

Per-image error pixel count

image_scale_factor

Array(float)

Per-image scale factor, if scaling/merging was performed

integrated_reflections

Array(int32)

Per-image count of integrated reflections

indexed_lattice_count

Array(int32)

Per-image count of indexed lattices

niggli_class

Array(uint8)

Per-image Niggli class identifier for indexed images; 0 if unavailable

pixel_sum

Array(int64)

Per-image sum of all valid pixels, excluding error/saturated pixels

image_scale_mosaicity

Array(float)

Scaling result: Image scale mosaicity [deg]

image_scale_b_factor

Array(float)

Scaling result: Image scale B factor [Angstrom^2]

image_scale_cc

Array(float)

Scaling result: Image scale CC

End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted.

Calibration message

Field name

Type

Description

Present in DECTRIS format

type

String

value “calibration”

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

data

Map(string -> Image)

Calibration map (only single pedestal array per message)

User data

In many cases there is an interest from facilities to forward more metadata, than available explicitly in the Jungfraujoch. For this reason two fields can be provided: header_appendix (sent with start message) and image_appendix (send with image message). To increase flexibility, both appendices can contain any valid JSON message. These appendices are serialized into string and stored in CBOR messages as user_data.

Notably for start message, user_data can contain more information (non-DECTRIS compliant metadata). Therefore user_data is serialized by Jungfraujoch as CBOR object. There is member user which contains header_appendix defined in OpenAPI of Jungfraujoch.

Notes on images and compression

  • Images are encoded as DECTRIS MultiDimArray with typed array tags:

    • For RGB: shape [3, height, width], type: u8

    • For grayscale: shape [height, width], type according to bit depth and sign (e.g., uint16 LE)

  • Compression:

    • Uncompressed: raw CBOR byte string

    • Bitshuffle+LZ4: tag with [“bslz4”, elem_size, bytes]

    • Bitshuffle+Zstandard: tag with [“bszstd”, elem_size, bytes]

Notes on typed arrays

Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays.

Common tags used in this protocol include:

  • float32 little-endian arrays for Array(float)

  • uint8 arrays for compact boolean/integer flags such as image_indexed

  • int32 little-endian arrays for per-image counts

  • int64 little-endian arrays for large per-image integer values

  • uint64 little-endian arrays for histograms

\ No newline at end of file diff --git a/CHANGELOG.html b/CHANGELOG.html new file mode 100644 index 00000000..b12c9f56 --- /dev/null +++ b/CHANGELOG.html @@ -0,0 +1 @@ + Changelog — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Changelog

1.0.0

1.0.0-rc.158

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (cos^3 of that angle) instead of cos^3(2*theta), so it is correct for a tilted detector and matches PyFAI solidAngleArray and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (XtalOptimizer) no longer silently ignores an imported PONI rot3 (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured rot3 through the full PONI rotation.

  • jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting \\); write processing outputs next to the input file, with a Browse button and independent _process.h5 / merged .mtz/.cif toggles; and show the determined space group in the merge-statistics window.

  • jfjoch_viewer: Connect to a broker over https (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow.

  • jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source.

  • rugnux: Accept an absolute -o output prefix in offline processing.

  • rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged).

  • rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing.

  • rugnux: De-novo symmetry robustness - don’t over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don’t flag twinning in holohedral Laue classes where no twin law can exist.

  • jfjoch_writer: Fold the refined beam centre into the NXmx detector translation vector too (not only the informational beam_center fields), so a reprocessed _process.h5 has a self-consistent refined geometry.

  • Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames.

  • Packaging: The self-contained Linux viewer .tgz now bundles cuFFT, so it runs without a system CUDA toolkit (.deb/.rpm are unchanged, distro-managed).

  • Docs: Documentation updated to match the current analysis code and CLI.

1.0.0-rc.157

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux: Rebrand the offline data-processing subsystem as rugnux and consolidate all offline analysis into the single rugnux binary - jfjoch_process is now rugnux, the former jfjoch_azint is now rugnux --azint-only, and jfjoch_scale is now rugnux --scale (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (--no-merge disables them), replacing the previous opt-in -M, --scale-merge.

  • rugnux: CLI fixes - default -N to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require --wavelength > 0, and correct the reproduced command line and --scale reference-cell handling.

  • rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics.

  • rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection’s rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below <I/sigma> = 1.

  • rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below --min-captured-fraction (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned).

  • rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved jfjoch prefix.

  • jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation.

  • jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a “reduce frame rate” warning.

  • jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint.

  • jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed).

  • jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types.

  • jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails.

  • jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak.

  • Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation.

  • CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*.

1.0.0-rc.156

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output.

  • jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed.

  • jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections.

  • jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw).

  • jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path.

  • jfjoch_process: Unify still and rotation processing on a single –force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental –reciprocal-profile option is removed.

  • jfjoch_process: Add EXPERIMENTAL ice-ring detection (–detect-ice-rings) that excludes ice reflections from scaling.

  • jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend.

  • jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata.

  • jfjoch_writer: Don’t write empty grid-scan position arrays when the dataset has no images.

  • Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don’t fit rather than aborting), and add the jfjoch_recompress tool.

  • jfjoch_viewer: Report “Multiple lattices detected” and grey out “Analyze dataset” on a live connection.

  • jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons.

  • CI: Build Windows (CUDA and non-CUDA) installers.

  • CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz.

1.0.0-rc.155

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D)

  • jfjoch_viewer: Some graphical improvements.

  • jfjoch_viewer: Simplify und unify data analysis settings.

  • jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker “dies” in the middle of data acquisition.

1.0.0-rc.154

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability)

1.0.0-rc.153

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing

  • jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files

  • jfjoch_broker: Add ROI calculation in non-FPGA workflow

  • jfjoch_broker: Fixes to TCP image pusher

  • jfjoch_broker: Remove NUMA bindings

  • jfjoch_broker: Improvements to indexing

  • jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values

  • jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors

  • jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs

  • jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results

1.0.0-rc.152

  • jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5)

  • jfjoch_viewer: Adjust Q bounds for azimuthal integration

  • jfjoch_azint: Add tool to do quick azimuthal integration

1.0.0-rc.151

  • jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname

1.0.0-rc.150

  • jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin.

  • jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins.

1.0.0-rc.149

  • XDS plugin: Fix HDF5 mutex to run on multiple processors

1.0.0-rc.148

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Improve azimuthal integration (add <I^2> calculation)

  • jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated)

  • jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin

1.0.0-rc.147

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015.

  • jfjoch_viewer: Add reciprocal space viewer

  • jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset

  • jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress)

1.0.0-rc.146

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension)

  • jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset.

  • jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling.

  • jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file.

  • jfjoch_scale: Dedicated tool for rescaling/merging existing data.

  • jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel.

WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being.

1.0.0-rc.145

This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

  • Default HDF5 writing mode is with VDS, not soft-links - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.

  • XDS plugin: Improve performance of VDS reading.

  • jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.

  • jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

1.0.0-rc.144

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M)

1.0.0-rc.143

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment

1.0.0-rc.142

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8

  • Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent

1.0.0-rc.141

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times

  • frontend: Fix selection of FFTW in indexing settings

1.0.0-rc.140

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection

  • jfjoch_broker: Minor bug fixes for rare conditions

  • jfjoch_process: Significantly improve performance

1.0.0-rc.139

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on /start

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on ‘/start’

  • jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite worklow preparation time

  • jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received

  • jfjoch_broker: Use nanoseconds internally for frame/image/readout time

  • jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished)

  • jfojch_broker: Improve profiling of different data analysis steps

  • jfjoch_broker: Record integration reflection count

  • jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds)

  • jfjoch_broker: Fix bug where ‘/wait_till_done’ got deadlocked

  • jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets

Breaking changes: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time.

1.0.0-rc.138

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time

  • jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations

  • jfjoch_broker: Goniometer axis name is converted to lowercase

  • jfjoch_broker: Fix bug, where wrong HTTP error codes were returned

  • jfjoch_process: Improve sigma estimation during merging (K. Takaba)

  • jfjoch_process: Modify spot finding thresholds

1.0.0-rc.137

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Better track time for each operation in the processing stack

  • jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress)

  • jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress)

  • jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0)

1.0.0-rc.136

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress).

1.0.0-rc.135

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4)

  • jfjoch_viewer: Improve image render performance

1.0.0-rc.134

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long

  • jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata

  • XDS plugin: Add generation of Jungfraujoch plugin for XDS

  • CI: Add tests with XDS and DIALS (xia2.ssx)

1.0.0-rc.133

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Use httplib for HTTP server instead of Pistache

  • jfjoch_broker: Drop OpenSSL support

  • jfjoch_broker: Base work for multi-lattice support in the future

  • jfjoch_broker: Improve recording time of data analysis steps

  • jfjoch_writer: Save per-image information about data analysis timing

  • Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib)

1.0.0-rc.132

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • Documentation: Fix equation rendering

1.0.0-rc.131

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS)

  • jfjoch_viewer: Fix calibration (pedestal) images being open flipped

  • jfjoch_process: Add space group detection (EXPERIMENTAL)

1.0.0-rc.130

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Rotation indexer has two retries if failes

  • jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot)

  • jfjoch_broker: Integration calculates background mask based on R2 radius

  • jfjoch_process: HDF5 files are not saved by default

1.0.0-rc.129

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files)

  • jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction

  • jfjoch_broker: Image pusher statistics are accessible via the REST interface

  • jfjoch_writer: Supports TCP image socket and for these auto-forking option

1.0.0-rc.128

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Handle properly reuse of image buffer locations

  • jfjoch_broker: Fix bug in counting idle slots

  • jfjoch_broker: Force obtuse angle for monoclinic cells

  • jfjoch_process: Change scaling refinement tolerance

1.0.0-rc.127

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Default EIGER readout time is 20 microseconds

  • jfjoch_broker: Multiple improvements regarding performance

  • jfjoch_broker: Image buffer allows to track frames in preparation and sending

  • jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer

  • jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets

  • jfjoch_writer: Fixes regarding properly closing files in long data collections

  • jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code

1.0.0-rc.126

This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg.

1.0.0-rc.125

This is an UNSTABLE release. This version adds scalign and merging. These are experimental at the moment, and should not be used for production analysis. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Improve logic on switching on/off spot finding

  • jfjoch_broker: Increase maximum spot count for FFBIDX to 65536

  • jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD)

  • jfjoch_process: Add scalign and merging functionality - program is experimental at the moment and should not be used for production analysis

  • jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection

  • jfjoch_writer: Save more information about each reflection

1.0.0-rc.124

This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123.

  • jfjoch_broker: Improve refection position prediction and Bragg integration code.

  • jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation.

  • jfjoch_writer: Fix saving mosaicity properly in HDF5 file.

  • jfjoch_viewer: Introduce high-dynamic range mode for images

  • jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%)

  • jfjoch_viewer: Zoom-in numbers have better readability

1.0.0-rc.123

This is an UNSTABLE release.

  • jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility

  • jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Add more color schemes for better image readability

  • HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable

  • HDF5: suppress warning if path (upstream group) doesn’t exists when checking if leaf exists

1.0.0-rc.122

This is an UNSTABLE release.

  • jfjoch_broker: Add thresholding to prefer shorter vectors after FFT

  • jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress)

  • jfjoch_broker: Update nlohmann::json to 3.12.0

  • jfjoch_viewer: Display file opening errors

  • jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available

1.0.0-rc.121

This is an UNSTABLE release.

  • jfjoch_broker: Report changes in the image buffer, so viewer doesn’t reload constantly

  • jfjoch_viewer: Improve performance of loading images

  • jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes

  • jfjoch_viewer: Auto-foreground calculated with histogram

  • jfjoch_viewer: Fix rare segmentation fault

1.0.0-rc.120

This is an UNSTABLE release.

  • jfjoch_broker: Improve performance of binary plot export

1.0.0-rc.119

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Add binary export of data analysis plots over OpenAPI

  • jfjoch_broker: Minor fixes to HTTP error handling

  • jfjoch_viewer: Prefer binary plots over JSON plots

  • jfjoch_viewer: Change foreground with F button + wheel

  • jfjoch_viewer: Change way how angles are displayed

  • jfjoch_viewer: Display resolution of the mouse cursor in top left corner

1.0.0-rc.118

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement.

  • jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations

1.0.0-rc.117

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add ROI results to the dataset info plots

  • jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment

1.0.0-rc.116

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add binning options in the context menu

1.0.0-rc.115

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Default spot finding settings can be configured via config JSON

  • jfjoch_viewer: FFT analysis of data in the dataset plot

1.0.0-rc.114

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Fix generating JPEG images with resolution estimation

1.0.0-rc.113

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Improve handling of rotation indexing

  • jfjoch_broker: More information saved in CBOR end message (WIP)

  • jfjoch_writer: Save rotation indexing lattice parameters and Niggli class

  • jfjoch_viewer: Remove (for now) primitive cell information

  • jfjoch_viewer: Use angle for dataset info plot for rotation scans

1.0.0-rc.112

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Experimental rotation (3D) indexing

  • jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values

1.0.0-rc.111

This is an UNSTABLE release.

  • jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment)

  • jfjoch_viewer: Fix auto contrast not refreshing image

1.0.0-rc.110

This is an UNSTABLE release.

  • jfjoch_broker: Add auto-contrast option for preview images

  • Frontend: Add logo image

  • jfjoch_viewer: Add logo image

  • jfjoch_viewer: For image chart allow to set min value to zero

  • jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure

  • jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there)

  • Documentation: Add logo image

1.0.0-rc.109

This is an UNSTABLE release.

  • jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard

  • jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors

1.0.0-rc.108

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN)

  • jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default

  • jfjoch_viewer: Fit to window of image is now default when size of the grid changes

1.0.0-rc.107

This is an UNSTABLE release.

  • jfjoch_viewer: Minor polishing of new functionality

  • jfjoch_broker: User NaN for empty azimuthal bins

1.0.0-rc.106

This is an UNSTABLE release.

  • jfjoch_viewer: Allow for multiple dataset info plots

  • jfjoch_viewer: Highlight current element in grid

1.0.0-rc.105

This is an UNSTABLE release.

  • jfjoch_viewer: Clean-up widgets slightly

  • jfjoch_viewer: Limit right panel to 600 pixels

  • jfjoch_viewer: Parse crystal symmetry type

  • jfjoch_viewer: Grid scan view takes color map and can be fit to zoom

1.0.0-rc.104

This is an UNSTABLE release.

  • jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier)

  • jfjoch_viewer: Display grid scan results in 2D (work in progress)

  • jfjoch_viewer: Improve auto-scaling on start of images (work in progress)

  • jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots

1.0.0-rc.103

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

  • jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates

1.0.0-rc.102

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

1.0.0-rc.101

This is an UNSTABLE release.

  • jfjoch_viewer: Auto load is better handling change of states

  • jfjoch_viewer: Fix DBus registration

  • jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update

  • jfjoch_viewer: Calculate ROI in a more efficient way

1.0.0-rc.100

This is an UNSTABLE release.

  • jfjoch_viewer: Fix dbus registration

  • jfjoch_viewer: Remove background slider for diffraction image

  • jfjoch_viewer: Adjustments for 2D azimuthal image viewer

1.0.0-rc.99

This is an UNSTABLE release.

  • jfjoch_broker: Fix output during mask data collection

1.0.0-rc.98

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization

1.0.0-rc.97

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask

  • jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems)

  • jfjoch_viewer: Add option to handle user pixel mask (experimental)

  • jfjoch_viewer: More options for ROI

  • jfjoch_viewer: Add window to display calibration

1.0.0-rc.96

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • jfjoch_broker: Remove PNG preview, no dependency on libpng

  • jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds)

  • jfjoch_viewer: Show data collection time in dataset tooltip

  • jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate)

  • jfjoch_viewer: Auto foreground value

  • Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds

  • Remove OpenBLAS dependency

1.0.0-rc.95

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • Add git-lfs to Rocky8 docker image

Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved.

1.0.0-rc.94

This is an UNSTABLE release.

  • FFTIndexer: Add limit on angles to avoid colinear vectors

  • Docker images: Add 3D Qt

  • Gitea: Fixes to the pipeline

1.0.0-rc.93

This is an UNSTABLE release.

  • CI: Fixes to Gitlab based pipeline

  • PCIe driver: Fix PCIe revision being hex number

1.0.0-rc.92

This is an UNSTABLE release.

  • jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample.

1.0.0-rc.91

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated

  • jfjoch_broker: Spot finding and integration predictions are ported to a GPU

  • jfjoch_broker: Estimate resolution

  • jfjoch_broker: Lattice search

  • jfjoch_broker: Many more improvements in image analysis

1.0.0-rc.90

This is an UNSTABLE release.

  • jfjoch_broker: for indexing min index spots for a viable cells can be changed via OpenAPI

  • jfjoch_viewer: Optional auto-reanalyze images

  • jfjoch_writer: Add option where no files at all are saved

  • Documentation: improvements

1.0.0-rc.89

This is an UNSTABLE release.

  • jfjoch_broker: Fix resolution estimation code

  • jfjoch_broker: Fix Wilson B-factor calculation code

  • jfjoch_viewer: Improve display of plots

  • jfjoch_viewer: Fix segmentation fault

  • jfjoch_viewer: Display missing metadata when using HTTP

  • jfjoch_viewer: Fix bug when opening the same file twice

1.0.0-rc.88

This is an UNSTABLE release.

  • jfjoch_viewer: Add resolution estimation to the image information

  • jfjoch_broker: Minor changes to resolution estimate routine

1.0.0-rc.87

This is an UNSTABLE release.

  • jfjoch_viewer: Display more image metadata (angle / exposure time)

  • jfjoch_viewer: Improve I/sigma and B-factor plots

  • jfjoch_broker: Estimate resolution based on visible spots

1.0.0-rc.86

This is an UNSTABLE release.

  • jfjoch_broker: Update logic when initializing detector to make it a bit more resilient

  • Gitea pipelines have nocuda option for all architectures

1.0.0-rc.85

This is an UNSTABLE release.

  • jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image

  • jfjoch_viewer: Handle spot count better in dataset info plots

  • jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled

1.0.0-rc.84

This is an UNSTABLE release.

  • jfjoch_broker: Write in log which detector is being initialized

  • Changes to automated build system

1.0.0-rc.83

This is an UNSTABLE release.

  • jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before)

  • CI: Fix script to generate python client

1.0.0-rc.82

This is an UNSTABLE release.

  • jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment)

  • Frontend: Minor fixes

  • Build scripts: Minor fixes to FFTW

1.0.0-rc.81

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing

  • jfjoch_broker: Add FFTW based indexer for CPU only indexing

  • jfjoch_broker: Enable saving X-ray fluorescence spectra

  • jfjoch_writer: Write total spot count (before filtering)

  • jfjoch_viewer: Add more information on source, sample, and buttom to show ice rings

  • jfjoch_viewer: Enable data processing inside the viewer

CI: Moving from Gitlab to Gitea at PSI

1.0.0-rc.80

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer

1.0.0-rc.79

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance

  • jfjoch_writer: Save nimages_per_trigger in detectorSpecific

1.0.0-rc.78

This is an UNSTABLE release.

  • jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore)

1.0.0-rc.77

This is an UNSTABLE release.

  • jfjoch_viewer: Display detector and dataset settings with tooltips

  • jfjoch_viewer: Clean excessive HDF5 warnings

  • jfjoch_viewer: Display unit cell

  • jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset

1.0.0-rc.76

This is an UNSTABLE release.

  • jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones

  • jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median)

  • jfjoch_writer: Save space group number (non-NXmx addition) in addition to name

  • jfjoch_viewer: Fix the bug on reading space_group as string

  • jfjoch_viewer: Add missing resolution labels on rings

  • jfjoch_viewer: Remove Q value from the status bar

1.0.0-rc.75

This is an UNSTABLE release.

  • jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being

1.0.0-rc.74

This is an UNSTABLE release.

  • jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing)

  • jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization

1.0.0-rc.73

This is an UNSTABLE release.

  • jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default

1.0.0-rc.72

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Refactor of indexing and geometry refinement code

  • jfjoch_broker: Handle space group/centering in refinement code

  • jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images

  • jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion

  • jfjoch_writer: Experimental saving of reflections

  • jfjoch_writer: Save space group name as string

  • jfjoch_viewer: Add profile radius and B-factor

  • jfjoch_viewer: Show 4 digits for wavelength

  • jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings)

  • FPGA: Use UDP destination port to distinguish between detector modules and data streams

  • FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up)

  • FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene)

1.0.0-rc.71

This is an UNSTABLE release.

  • jfjoch_broker: Remove resolution estimation via machine learning

  • jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers

  • jfjoch_viewer: Fix error when HDF5 files with indexing results couldn’t be loaded on a machine without GPU

1.0.0-rc.70

This is an UNSTABLE release. This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement

  • jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards)

  • jfjoch_broker: Enable geometry refinement - first try (work in progress)

  • jfjoch_viewer: Fix deadlock when opening HTTP connections

  • jfjoch_viewer: Display rings as ellipses with detector tilt

  • jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image

  • jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3)

  • Add Google Ceres a non-linear least-square optimization library to Jungfraujoch

  • Add experimental detector calibration routines (for LaB6)

  • Improve documentation on the ZeroMQ writer notification socket and detector geometry

1.0.0-rc.69

This is an UNSTABLE release.

  • jfjoch_viewer: Metadata can be modified for an open dataset (no option to save)

  • jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments

  • jfjoch_viewer: Resolution rings not enabled by default

  • jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file

1.0.0-rc.68

This is an UNSTABLE release.

  • jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector

  • jfjoch_broker: Default detector settings can be configured for each detector separately

  • jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings

  • jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing

  • jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter

  • jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI

1.0.0-rc.67

This is an UNSTABLE release.

  • jfjoch_broker: Enable SSL

  • jfjoch_broker: Wilson B-factor only provided is fit is relatively OK (R^2 > 0.3); this will be refined much more in the future

1.0.0-rc.66

This is an UNSTABLE release.

  • jfjoch_broker: Indexers operate as thread pool, which is operating

  • jfjoch_viewer: Increase interval between loading images + fix too many verbose messages

1.0.0-rc.65

This is an UNSTABLE release.

  • jfjoch_broker: Print information regarding used image pushers

  • jfjoch_viewer: Allow syncing with Jungfraujoch server

  • OpenAPI: Clarify licensing terms in the file

1.0.0-rc.64

This is an UNSTABLE release.

  • jfjoch_broker: Fix issue in receiver light with very long preparation time for threads

  • jfjoch_broker: Add verbose option

  • jfjoch_broker: Don’t trigger pedestal if critical settings are not changed when loading detector settings

  • jfjoch_broker: Detector left in busy state when detector settings were improper

  • jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times

  • jfjoch_lite_perf_test: Add verbose option

1.0.0-rc.63

This is an UNSTABLE release.

  • jfjoch_broker: Save NX/NY for grid scan result

  • jfjoch_broker: Add processing time to CBOR output and plot

  • jfjoch_writer: Add processing time to data file

1.0.0-rc.62

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where low resolution spots were not counted properly

  • jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count

  • jfjoch_broker: Add more spot count information to CBOR

  • jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles

1.0.0-rc.61

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector

  • jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results

  • jfjoch_broker: High resolution limit for low res. spot counting can be adjusted

1.0.0-rc.60

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released

  • jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices

  • jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength

1.0.0-rc.59

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock.

  • jfjoch_writer: Verbose option for debugging.

1.0.0-rc.58

This is an UNSTABLE release.

  • jfjoch_viewer: Fix memory leak

  • jfjoch_writer: Add detector_number/serial_number to master file

1.0.0-rc.57

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible

  • jfjoch_viewer: “Movie” mode

1.0.0-rc.56

This is an UNSTABLE release.

  • jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation

1.0.0-rc.55

This is an UNSTABLE release.

  • jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation

1.0.0-rc.54

This is an UNSTABLE release.

  • jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not)

  • jfjoch_broker: Don’t remove shared memory area when deactivating detector

  • jfjoch_writer: Save writer release

  • jfjoch_viewer: Increase time for the messages in the status bar

1.0.0-rc.53

This is an UNSTABLE release.

  • PCIe driver: Imperfect solution for RHEL 9.5+ changes

  • jfjoch_writer: Fix to angle containers for AutoProc compatibility

  • jfjoch_fpga_test: Use consecutive number for devices, not interleaved

1.0.0-rc.52

This is an UNSTABLE release.

  • jfjoch_viewer: Use warmer colors to distinguish from AareGUI

  • jfjoch_viewer: Minor adjustments to DBus setting image number

  • jfjoch_broker: Fix in low resolution spot count plotting

1.0.0-rc.51

This is an UNSTABLE release.

  • jfjoch_broker: Send preview in PNG format

  • jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range

  • jfjoch_broker: Provide ML resolution estimation in scan result

  • jfjoch_broker: Allow removing beam center in web preview

1.0.0-rc.50

This is an UNSTABLE release.

  • The release fixes some of many bugs introduced in recent releases

  • jfjoch_viewer: display predictions for indexed cells

1.0.0-rc.49

This is an UNSTABLE release.

  • jfjoch_broker: Handle sample temperature (K) and ring current (mA) to metadata

  • jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current

1.0.0-rc.48

This is an UNSTABLE release.

  • jfjoch_broker: fix the bug when a unit cell was not exported for a scan result.

1.0.0-rc.47

This is an UNSTABLE release.

  • jfjoch_viewer: fix dbus service path

  • jfjoch_writer: fix CBF/TIFF writing

1.0.0-rc.46

This is an UNSTABLE release.

  • jfjoch_viewer: remove dependency on image analysis

1.0.0-rc.45

This is an UNSTABLE release.

  • jfjoch_broker: Detector list returns pixel size (mm)

1.0.0-rc.44

This is an UNSTABLE release.

  • jfjoch_broker: more general definition of scan result export

Braking changes:

  • It removes additions to OpenAPI from 1.0.0-rc.43

  • It makes changes to the “unit_cell” definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations.

1.0.0-rc.43

This is an UNSTABLE release.

  • jfjoch_broker: Export grid scan results into a single data structure

1.0.0-rc.42

This is an UNSTABLE release.

  • jfjoch_broker: Add pixel_sum to CBOR output.

  • jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine

  • jfjoch_writer: Save pixel_sum

1.0.0-rc.41

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction

  • jfjoch_broker: Export GPU count via OpenAPI

  • jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector

  • FPGA: Increase the number of integration bins to 2048

1.0.0-rc.40

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures

  • jfjoch_broker: Improve metadata for rotation data collection

  • jfjoch_broker: Better handling of plotting

  • jfjoch_broker: FFT based indexing

  • jfjoch_broker: Integration, first try, results not saved at the moment

  • jfjoch_broker: Internal improvements in image handling

  • jfjoch_writer: Multiple adjustments adapt to changes in this release for new features

  • jfjoch_writer: New state management model to improve clarity of error reporting

  • jfjoch_viewer: Remote control via DBus

  • Frontend: Multiple adjustments for new features

  • Frontend: Grid scan plots

WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken.

1.0.0-rc.39

  • FPGA: Bugfix for pixel masked for data analysis if summation was on

  • jfjoch_viewer: Fix segmentation fault when cursor was outside of image

1.0.0-rc.38

  • jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via RES

  • jfjoch_broker: Neural net model can use all 4 quadrants of the detector

  • jfjoch_broker: For EIGER image time can be provided through /start

  • jfjoch_viewer: Add image list option

  • jfjoch_viewer: Drawing circular ROIs with shift

  • jfjoch_viewer: Enable image summation

  • jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer

1.0.0-rc.37

  • jfjoch_broker: Make locking rules more flexible

  • jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors

  • jfjoch_viewer: Add status bar

1.0.0-rc.36

This is UNSTABLE release. Wait for new version to use in a production environment.

  • jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code)

  • jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation)

  • jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data

  • jfjoch_writer: Increase buffering to 1000 images in the receiver

  • jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5

1.0.0-rc.35

This is UNSTABLE release, not properly tested. Wait for new version for using production.

  • jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting.

  • jfjoch_writer: Save EIGER energy threshold

  • jfjoch_writer: Add /entry/sample/goniometer for compatibility with eiger2cbf program

1.0.0-rc.34

This is UNSTABLE release - introducing new features, but not properly tested. Wait for new version for using production.

  • jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings)

  • jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells

  • jfjoch_broker: Handle situation when detector doesn’t want to gracefully stop (to be tested)

  • jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots

  • jfjoch_viewer: Can extract data analysis results from “legacy” format

  • jfjoch_viewer: Display dataset name

  • FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels

  • FPGA: Add pixel sum to module statistics

  • FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI

  • FPGA: Spot finder is back to full dynamic range (24-bit)

  • FPGA: More debug features for internal FIFOs

Known issues:

  • ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts.

  • EIGER data acquisition has an issue that is currently debugged

1.0.0-rc.33

  • jfjoch_broker: Fix issue with EIGER settings being loaded improperly

1.0.0-rc.32

  • jfjoch_broker: Refactor code for azimuthal integration for further improvements

  • jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later)

  • jfjoch_writer: Fix too much verbose information

  • FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20)

1.0.0-rc.31

This is UNSTABLE release - introducing many features, but still needs more testing. Expecting soon to put bugfix release.

  • jfjoch_writer: Allow to enable overwriting existing files (not enabled by default)

  • jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default)

  • jfjoch_viewer: Image viewer work early test version

  • jfjoch_broker: Fixes to counting packets per dataset/image

  • jfjoch_broker: Image buffer is accessible for outside to check images

  • jfjoch_broker: error/saturated pixels and dedicated ROI “beam” can be tracked online

  • jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU

  • jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation

  • jfjoch_broker: Fix bug in EIGER initializing

  • jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot

  • PCIe driver: Add PCIe link speed and width

  • FPGA: Improve counting error/saturated/min/max pixels

  • FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas

  • FPGA: Require Vivado 2022.2

Warning: There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency.

1.0.0-rc.30

  • jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to “loose” images within the writer

1.0.0-rc.29

  • jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU

  • jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file

  • jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info)

  • jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly

Warning - two breaking changes to OpenAPI:

  • Handling of ROIs is through /config/roi path only for both circle and box ROIs, path in /roi are no longer accessible

  • broker_status structure introduced in 1.0.0-rc.28 has member message and not error_message to allow handling info/warning messages as well

1.0.0-rc.28

  • jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI

  • jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not complaint to OpenAPI specs

  • jfjoch_test: remove header that failed when CUDA is absent during compilation

  • frontend: add soft trigger button in data collection tab

  • frontend: show error message when in error state

  • CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF)

1.0.0-rc.27

  • jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation

  • jfjoch_broker: update pistache web server version

  • jfjoch_writer: minor changes to republish logic

  • Improvements to documentation

1.0.0-rc.26

  • jfjoch_broker: implement ZeroMQ stream for image metadata information

  • jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent

  • jfjoch_broker: add crystal lattice plots

  • jfjoch_broker: remove empty bins from the plots

  • jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU “long” summation

  • jfjoch_broker: Fix bug when mean background estimation / indexing rate where affected by previous experiment

  • jfjoch_writer: fix missing “-w” parameter

  • jfjoch_writer: temporary files have “.tmp” suffix

  • jfjoch_writer: refactor logic for watermarks

  • jfjoch_writer: report on internal FIFO utilization

  • jfjoch_writer: clean-up naming for azimuthal integration and background estimate

  • jfjoch_writer: write final background estimate and indexing rate in the master file

  • tools/: remove unnecessary tools, make naming consistent

  • CBOR: Add indexing rate and background estimate to end message

  • CBOR: Clean-up documentation

1.0.0-rc.25

  • Updates to documentation

  • License set to GPLv3 / OHL-S

  • Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms

  • Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5

1.0.0-rc.24

New FPGA functionality:

  • EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit “real” depth is 23-bit + 1-bit signed)

  • Output possible to 8, 16 and 32-bit data

  • Threshold is applied before summation

  • Pixel mask can be applied on FPGA

  • Mark pixels with ADC content = 0 as bad pixels

  • FPGA stores semantic version information (access via /sys/class/misc/jfjoch…/version)

New software functionality:

  • Long summation (above 256 frames) done on CPU

  • Mechanism to save arbitrary data to HDF5 file

  • ZeroMQ preview has option to send start message

  • Rework pixel mask + add statistics displayed in web interface

Bug fixes:

  • Web frontend: Update preview image automatically during data acquisition

  • jfjoch_broker: Error handling if CUDA driver is not installed

  • jfjoch_broker: Correctly update progress during pedestal

  • jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF

  • jfjoch_action_test: enable HLS simulation

Documentation improvement and placement in a dedicated directory

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS.html b/CPU_DATA_ANALYSIS.html new file mode 100644 index 00000000..3884d34a --- /dev/null +++ b/CPU_DATA_ANALYSIS.html @@ -0,0 +1 @@ + CPU-side crystallographic data analysis (Jungfraujoch) — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

CPU-side crystallographic data analysis (Jungfraujoch)

This document describes the crystallographic algorithms implemented in Jungfraujoch for CPU- and GPU-side real‑time and near‑real‑time data analysis.

Scope. The pipeline covered here comprises:

  1. geometry mapping and corrections,

  2. azimuthal integration (powder/radial profiles),

  3. Bragg spot finding (strong pixels → connected components → spot descriptors),

  4. indexing (still and rotation modes),

  5. Bravais lattice / centering inference,

  6. geometry and lattice refinement,

  7. reflection prediction (still and rotation),

  8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free),

  9. scaling and merging,

  10. merge-level error modelling and outlier rejection,

  11. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref).

References

The methods are inspired and reuising solutions implemented in:

  • W. Kabsch, “XDS”, Acta Cryst. D66 (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts).

  • W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, Acta Cryst. D66 (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors).

  • T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts).

  • J. Kieffer & J. P. Wright, “PyFAI: a Python library for high performance azimuthal integration on GPU”, Powder Diffraction 28 (2013), S339-S350 (detector geometry definition, azimuthal integration)

  • H. Powell, “The Rossmann Fourier autoindexing algorithm in MOSFLM”, Acta Cryst. D55 (1999), 1690-1695 (FFT indexing) (list is not exhaustive)

1. Geometry, reciprocal-space mapping, and basic quantities

1.1 Coordinate conventions

For a pixel coordinate \((x,y)\) (in pixels), Jungfraujoch converts to a laboratory direction vector via:

  1. shift by direct-beam position \((x_\mathrm{beam}, y_\mathrm{beam})\),

  2. scale by pixel size \(p\) (mm),

  3. set detector distance \(D\) (mm),

  4. apply detector orientation rotation \(R_\mathrm{det}\) (PyFAI-like parameterization).

The unnormalized detector coordinate (mm) is: \( \mathbf{r}_\mathrm{det}(x,y) = \begin{pmatrix} (x-x_\mathrm{beam})p\\ (y-y_\mathrm{beam})p\\ D \end{pmatrix}. \)

The lab-frame vector is: \( \mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. \)

Let the incident wavevector magnitude be \(k = 1/\lambda\) in Å\(^{-1}\), and define: \( \mathbf{S}_0 = (0,0,k). \)

The reciprocal-space scattering vector associated with pixel \((x,y)\) is: \( \mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. \)

This \(\mathbf{s}\) is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement.

1.2 Two-theta, azimuth, resolution and \(q\)

The scattering angle \(2\theta\) is computed from \(\mathbf{r}_\mathrm{lab}\) via: \( 2\theta = \arctan\!\left(\frac{\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2}}{z_\mathrm{lab}}\right). \)

Resolution (Å) at a pixel is: \( d = \frac{\lambda}{2\sin(\theta)} = \frac{\lambda}{2\sin(2\theta/2)}. \)

The magnitude \(q = 2\pi/d\) is used for radial binning and ice-ring handling.

1.3 Distance from the Ewald sphere

For a reciprocal lattice point \(\mathbf{p}\) (Å\(^{-1}\)), define: \( \Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. \) Jungfraujoch uses \(|\Delta_\mathrm{Ewald}|\) as an operational proxy for excitation error. This appears in:

  • still prediction (accept if \(|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}\)),

  • profile radius estimation (see §11.1),

  • still partiality option in scaling/merging (§10.2).

2. Azimuthal integration (radial profiles)

Azimuthal integration produces a radial profile \(I(q)\) or \(I(d)\) by histogramming pixels into radial bins. Pixels are not split across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 \(\phi\) sectors (azim_bins, --azim-phi-bins), giving a 2D \(q\times\phi\) profile that exposes azimuthal anisotropy such as detector shadowing or sample texture.

2.1 Histogram estimator

Let bin index \(b(x,y)\) be precomputed from \(q(x,y)\) (or equivalently from \(d(x,y)\)) and, when \(\phi\) sectors are enabled, the azimuth \(\phi(x,y)\) — so \(b = b_q + b_\phi B_q\). For each bin \(b\):

  • accumulate corrected intensity and its square: \( S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, \)

  • and count: \( N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. \)

The profile reports both the mean \(\bar{I}_b = S_b / N_b\) (when \(N_b>0\)) and a per-bin sample standard deviation \(\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}\) (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded.

2.2 Corrections applied

Two standard corrections are available:

(i) Solid angle / geometric correction. A flat pixel’s solid angle falls off with the incidence angle \(\alpha\) between the scattered ray and the detector normal. With the in-plane detector offsets \(u=(x-x_\mathrm{beam})p\) and \(v=(y-y_\mathrm{beam})p\) (§1.1) and detector distance \(D\), \( \cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad C_\Omega = \cos^3\alpha, \) applied — like the polarization term below — as a divisor (intensities are scaled by \(1/\cos^3\alpha\)), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because \(\alpha\) is evaluated in the detector’s own frame it is invariant under detector tilt (\(\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}\)), matching PyFAI’s solidAngleArray and MAX IV azint. It reduces to the commonly quoted \(\cos^3(2\theta)\) form only for an untilted detector, where the incidence angle coincides with the scattering angle.

(ii) Polarization correction. With polarization coefficient \(P\) (beamline dependent) and azimuth \(\phi\): \( C_\mathrm{pol}(2\theta,\phi) = \frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), \) applied as a divisor to intensities (i.e. scale by \(1/C_\mathrm{pol}\)) when enabled.

2.3 Background estimate for profiles

A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid \(Q\) window (default \(2\pi/5\) to \(2\pi/3\) Å\(^{-1}\)). This background is used for monitoring and diagnostics; it is not the same as the local Bragg-spot background used in summation integration (§9.2).

3. Spot finding (strong pixels → Bragg spots)

Spot finding is a two-stage process:

  1. Strong-pixel selection using intensity and/or local signal-to-noise criteria.

  2. Connected-component labeling (CCL) to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction.

3.1 Strong-pixel detection by local statistics

For each pixel \(i\) with value \(v_i\), consider a square window (nominally \(31\times 31\) pixels) around it. Let the window contain \(n\) valid pixels (excluding masked/bad/saturated), and define: \( \Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. \)

To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: \( \Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. \)

A variance-like quantity proportional to \(n'^2\) is formed: \( V = n'\Sigma_2' - (\Sigma')^2, \) and the deviation-from-mean quantity: \( \Delta = v_i n' - \Sigma'. \)

A pixel is considered strong if:

  • it is above a photon/count threshold, and

  • its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and

  • \(\Delta>0\), and

  • the squared deviation exceeds a scaled variance: \( \Delta^2 > V\cdot T^2, \) where \(T\) is the configured signal-to-noise threshold.

This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast.

Special cases:

  • saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots),

  • invalid pixels are never strong.

3.2 Resolution and ice-ring handling

Spot finding can be restricted to a resolution range \([d_\mathrm{high}, d_\mathrm{low}]\) by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6).

A single per-image ice-ring score is derived from the azimuthally-integrated radial profile: for each hexagonal-ice powder ring (positions \(d\) from Moreau et al., Acta Cryst D77, 2021), the profile intensity at the ring is divided by a smooth background estimated from the whole profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring’s ratio is reported (1 = no ice, \(>1\) = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring’s own edge (offline processing defaults to a fine 0.01 1/Å spacing, --azim-q-spacing, so the rings are well resolved). (A significance/z-score was considered but is uninformative here: with many photons any real ice ring is highly significant, so the discriminating quantity is the ice magnitude, i.e. this ratio.) It is stored per image (ice_ring_score, HDF5 /entry/MX/iceRingScore) as a monitoring quantity, distinct from the merge-time ice masking, which is data-driven from the per-ring merged CC1/2.

A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in \(1/d\) (or \(q\)) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low.

3.3 Connected-component labeling (CCL)

Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with:

  • centroid \((x,y)\) (often intensity-weighted),

  • pixel count (spot size),

  • integrated spot intensity proxy (sum of pixel values),

  • resolution \(d\) at the centroid (or mean over pixels),

  • and quality flags (e.g. ice-ring classification).

Spot-level filters include minimum/maximum pixel count and resolution limits.

4. Indexing overview

Indexing maps observed reciprocal-space vectors \(\mathbf{s}_i\) to a lattice such that: \( \mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, \) with integer \((h_i,k_i,l_i)\).

Jungfraujoch supports two complementary indexing strategies:

  1. FFT-based indexing (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples.

  2. Fast-feedback indexing (TORO-like): requires an approximate unit cell; optimized for speed and feedback.

Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return more than one lattice per image (multi-lattice indexing; see §5.4).

4.1 Indexed-spot decision (inlier test)

Given a trial lattice with direct basis vectors \(\mathbf{a},\mathbf{b},\mathbf{c}\) (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: \( h_f = \mathbf{s}\cdot\mathbf{a},\quad k_f = \mathbf{s}\cdot\mathbf{b},\quad l_f = \mathbf{s}\cdot\mathbf{c}. \) Let \((h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))\) and define the fractional residual: \( \delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. \) A spot is indexed if \(\delta^2 < \tau^2\), where \(\tau\) is the configured tolerance.

For indexed spots, the reciprocal lattice point \(\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*\) is used to compute \(\Delta_\mathrm{Ewald}(\mathbf{p})\) (stored as a diagnostic and later used in profile-radius estimation).

5. FFT indexing (unknown unit cell)

FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms.

5.1 Directional projections and histograms

Choose a set of unit vectors \(\{\mathbf{u}_d\}\) on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction \(d\), form a histogram in the scalar projection: \( t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. \)

Bin width is chosen approximately as: \( \Delta t \approx \frac{1}{2 L_\mathrm{max}}, \) where \(L_\mathrm{max}\) is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum \(q\) used (set by a high-resolution cutoff for indexing).

5.2 FFT peak picking and candidate vectors

For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along \(\mathbf{u}_d\). Each direction yields a candidate real-space length \(L\) chosen not by raw magnitude but by maximum prominence above a running-mean local background (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to \(L\ge L_\mathrm{min}\).

Candidate vectors are \(\mathbf{v}_d = L_d\,\mathbf{u}_d\).

A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak.

5.3 Lattice reduction and cell candidates

Triples of candidate vectors are combined to form candidate bases \((\mathbf{A},\mathbf{B},\mathbf{C})\), each reduced to its Niggli-reduced cell (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a widened fallback anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses.

5.4 Robust refinement and best-cell selection

Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked:

  1. more indexed spots wins — unless two candidates index within ~10 % of each other, in which case

  2. the smaller-volume cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then

  3. the smaller refinement score, then the spot count again.

Selection is not limited to a single lattice: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded.

An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant).

7. Geometry and lattice refinement

Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices.

7.1 Parameterization

The refinement jointly optimizes, depending on mode and constraints:

  • beam center \((x_\mathrm{beam}, y_\mathrm{beam})\),

  • detector distance \(D\),

  • detector tilt angles (two-angle model; third rotation often held at 0),

  • rotation axis direction (for rotation datasets),

  • crystal orientation (a global rotation),

  • unit-cell parameters, with constraints determined by inferred crystal system.

By default only the beam center, unit cell and crystal orientation are refined; the detector distance, tilt angles and rotation-axis direction are held fixed unless explicitly enabled. A lighter orientation-only mode refines just the crystal orientation (with a weak small-rotation prior on the poorly-determined out-of-plane component), for stills whose geometry is already trusted.

For higher symmetries, constraints are enforced, e.g.

  • cubic: \(a=b=c,\ \alpha=\beta=\gamma=90^\circ\),

  • tetragonal: \(a=b\),

  • hexagonal: \(a=b,\ \gamma=120^\circ\),

  • monoclinic (unique axis \(b\)): \(\alpha=\gamma=90^\circ\), \(\beta\) refined.

7.2 Residuals and objective

For each indexed spot assigned integer \((h,k,l)\), compute:

  • observed reciprocal vector \(\mathbf{s}_\mathrm{obs}\) from its detector position and current geometry,

  • predicted reciprocal vector \(\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})\).

Residual is: \( \mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. \)

A non-linear least squares solver minimizes \(\sum \|\mathbf{r}\|^2\) over all selected inlier spots.

7.3 Rotation datasets: bringing observations to a common reference frame

For oscillation/rotation data, each image corresponds to a rotation angle \(\phi\) about an axis \(\mathbf{m}_2\). Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: \( \mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, \) with \(R(\phi)\) constructed from the axis-angle representation of the goniometer model. The angle \(\phi\) is taken at the centre of each frame’s oscillation (the frame angle plus half the oscillation width).

7.4 Multi-stage tightening of inlier tolerance

Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance \(0.3\to0.2\to0.1\)), which stabilizes convergence when starting from imperfect indexing and approximate geometry.

8. Reflection prediction

Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model.

8.1 Enumerating reciprocal lattice points

For a maximum resolution \(d_\mathrm{min}\), accept \((h,k,l)\) such that: \( \lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. \)

8.2 Still prediction (excitation-error cutoff)

For still images, the diffracting condition is approximated by an excitation-error cutoff: \( \left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. \) Accepted reflections are projected to the detector by intersecting the diffracted direction \(\mathbf{S}=\mathbf{S}_0+\mathbf{p}\) with the detector plane, using the current geometry.

When the beam has a finite energy bandwidth, this window is broadened radially per reflection: the cutoff is combined in quadrature with a bandwidth smear, \(\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}\), where \(\sigma_\mathrm{bw}\propto|p_z|\) (the reciprocal-space depth along the beam, growing as \(\sim 1/d^2\)). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same \(\sigma_\mathrm{bw}\) is deconvolved from the measured profile radius (§11.1), so it is not double-counted.

8.3 Rotation prediction (Laue equation + partiality model)

For rotation/oscillation datasets, Jungfraujoch solves for rotation angles \(\phi\) where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define:

  • rotation axis unit vector \(\mathbf{m}_2\),

  • \(\mathbf{S}_0\) incident vector,

  • \(\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)\).

A key quantity is: \( \zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad \mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, \) which also appears in XDS as the Lorentz component linked to the rotation axis.

A Gaussian mosaicity model yields a partiality fraction over an oscillation width \(\Delta\phi\):

\( P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], \)

with mosaicity \(\sigma_M\) in radians.

Reflections are predicted if they meet minimum \(\zeta\) and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area.

8.4 Systematic absences (centering)

Systematic absences are applied at least at the centering level (prior to full space-group symmetry). For centering symbol \(C\):

  • \(I\): absent if \(h+k+l\) odd,

  • \(A\): absent if \(k+l\) odd,

  • \(B\): absent if \(h+l\) odd,

  • \(C\): absent if \(h+k\) odd,

  • \(F\): absent if any of \(h+k, h+l, k+l\) is odd,

  • \(R\): absent if \((-h+k+l)\bmod 3 \ne 0\),

  • \(P\): no centering absences.

9. 2D Bragg integration (profile fitting over a three-ring ROI)

Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The default extraction is profile fitting (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection \((I,\sigma,\text{partiality},d)\), so scaling, the rotation combine (§10.6) and merging consume either unchanged.

9.1 Regions of interest

For each predicted reflection at \((x_p,y_p)\), define three radii:

  • \(r_1\): inner signal radius,

  • \(r_2\): inner background radius,

  • \(r_3\): outer background radius.

Pixels are classified by their squared distance \(r^2=(x-x_p)^2+(y-y_p)^2\):

  • signal region: \(r^2 < r_1^2\),

  • background annulus: \(r_2^2 \le r^2 < r_3^2\).

Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk (\(r<r_2\)) of any other predicted reflection are removed from this reflection’s background annulus, so a neighbouring spot cannot leak into the background estimate.

9.2 Box summation (seed and fallback)

Let:

  • \(S = \sum I(x,y)\) over signal pixels,

  • \(n_S\) = number of valid signal pixels,

  • \(B = \sum I(x,y)\) over background pixels,

  • \(n_B\) = number of valid background pixels.

Background per pixel and integrated intensity: \( \hat{b} = \frac{B}{n_B},\qquad \hat{I} = S - n_S \hat{b}, \) with a Poisson-like uncertainty \(\sigma(\hat{I})=\max\!\big(1,\ r_\sigma\hat{I},\ \sqrt{S}\big)\), i.e. \(\sqrt{S}\) floored both at 1 and at a small fraction \(r_\sigma\) of the intensity. A reflection is accepted as “observed” only if all signal pixels were valid and \(n_B\) exceeds a minimum. This box sum is the classical estimator; it is used directly with --integrator boxsum, and otherwise seeds the profile fit below.

For the profile-fit path on broadband (still) data, the background mean is additionally computed with a single high-outlier reject (drop ring pixels above \(\hat{b}+3\sqrt{\hat{b}}\), then recompute): a bandwidth-streaked high-resolution spot or a close neighbour can leak into the ring and bias the mean high, over-subtracting and driving weak high-resolution intensities negative. A clean Poisson background is essentially unchanged by the cut. The reject is not applied to plain box summation (--integrator boxsum) or to monochromatic/rotation data.

9.3 Profile-fitted extraction (default)

A fixed signal disk captures a width-dependent fraction of each spot, which puts a multiplicative floor on the per-observation precision of strong reflections and weights weak reflections poorly. Profile fitting removes this by extracting each intensity against a fitted spot shape, without needing reference intensities. Per frame:

  1. Seed. Box-sum every reflection (§9.2) to get a rough intensity and observed centroid, and select strong spots (significance \(\ge 5\)).

  2. Build the profile. For gaussian (the default) the width is taken per resolution shell from the measured second moment of the strong spots (shell-dependent because spot size grows with resolution); the intrinsic spot is essentially round in the detector plane (per-detector-region and crystal-anisotropy profiles were evaluated and add nothing — the real crystal anisotropy lives in the discarded rocking direction). For empirical the profile is instead the averaged, centroid-aligned, background-subtracted pixel grid of the shell’s strong spots. Either way the profile is then rebuilt for each reflection, centred on its sub-pixel predicted position (the noise-free geometric centre, not the observed centroid) and, where needed, elongated only along the radial direction (away from the beam centre) — because two effects stretch a spot radially but not tangentially:

    • a finite energy bandwidth smears each spot by \(\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}\) (\(R_\mathrm{px}\) = distance from the beam centre, large at high resolution), and

    • sensor parallax — the depth over which a photon converts in a thick Si/CdTe sensor — adds a term \(\propto\tan^2(2\theta)\) (material- and energy-dependent), plus, on the monochromatic path, a small fixed weak-spot capture term.

    These combine as \(\sigma^2_\mathrm{radial}=\sigma^2_\mathrm{intrinsic}+\sigma_\mathrm{bw}^2+c_\mathrm{par}\tan^2(2\theta)\) (tangential unchanged), on a grid grown to hold the streak — capturing it without the tangential background an isotropic widening would add.

  3. Fit (Kabsch). With profile \(P\), background \(B\) and the shell variance model, the intensity and its uncertainty are \( I = \frac{\sum P\,(c-B)/v}{\sum P^2/v},\qquad \sigma = \sqrt{\frac{1}{\sum P^2/v}},\qquad v = B + \max(I,0)\,P, \) where \(c\) is the pixel value and the de-biased variance \(v\) (background plus model signal, rather than the down-fluctuating observed count) is iterated (a few passes). As a guard, if the profile intensity runs away from the box-sum seed (by more than ~10 box-sum \(\sigma\)) it falls back to the seed, and the variance floors the background at \(1/12\) (the integer-binning pixel-variance floor). The rotation/excitation partiality is carried exactly as in the box-sum path.

The integrator is selected by --integrator boxsum|gaussian|empirical (default gaussian).

9.4 Lorentz–polarization factor handling

For integrated reflections, polarization correction can be applied as a multiplicative correction to the reflection scale via the geometry-based polarization term (§2.2). A Lorentz-like factor is carried as rlp in predictions, and used during scaling/merging (§10).

10. Scaling and merging

After per-image integration, Jungfraujoch scales observations and merges them into unique reflections. The design is intentionally compatible with XDS/XSCALE concepts, and handles both still and rotation data.

10.1 Observation model

For an observation \(j\) of a unique reflection \(h\) on image (or image group) \(i\), the predicted measured intensity is modeled as: \( I_{ij} \approx G_i \, L_{ij}\, P_{ij}\, I_h, \) where:

  • \(G_i\) is the image scale factor,

  • \(L_{ij}\) is a Lorentz-like / geometry factor (stored as rlp or derived),

  • \(P_{ij}\) is a partiality term (model-dependent),

  • \(I_h\) is the merged (true) intensity parameter for that unique reflection.

A least-squares objective is minimized: \( \sum_{ij} \left(\frac{I_{ij}^{\mathrm{pred}} - I_{ij}^{\mathrm{obs}}}{\sigma_{ij}}\right)^2 \) solved by robust (Cauchy) weighted least squares, with optional post-fit smoothing of the per-frame scales for rotation series (§10.3).

10.2 Partiality models

The partiality applied is fixed by the data type and scaling stage, not chosen from a user menu:

  1. Rotation partiality (XDS-like; see §8.3), used for the per-frame scaling of rotation partials: \( P_{ij} = \frac{1}{2}\left[ \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) \right]. \) The mosaicity \(\sigma_{M,i}\) is measured once per image at indexing (MLE, §11.2) and held fixed during scaling — only smoothed in frame order (§10.3), never re-refined (it is degenerate with the scale \(G\); §11.2).

  2. Unity (\(P_{ij}=1\)): used for the scale-on-fulls refit (§10.6), where each observation is already a complete reflection.

  3. Fixed: use the per-reflection partiality carried from prediction. Still/serial images are predicted with \(P=1\), so their scaling is effectively unity/fixed — there is no excitation-error still-partiality model.

Reflections below a minimum partiality can be rejected from merging to avoid unstable corrections.

10.3 Smoothing of per-frame scales

The per-frame scales \(G_i\) are fit by robust (Cauchy) inverse-variance-weighted ratios; there is no explicit \(G\approx1\) prior. For rotation datasets, optional smoothing enforces the expectation that scale and mosaicity vary slowly across a sweep: after the per-frame fit, \(\log G_i\) (and the mosaicity) are replaced by a centred moving average over a window spanning a configurable rotation range (XDS DELPHI-like; --smooth-g, default 5° for rot3d, off otherwise). It is a post-fit smoothing pass, not a curvature penalty inside the least-squares objective.

10.4 Merging estimator

After refinement, corrected observations are formed: \( I^{\mathrm{corr}}_{ij} = \frac{I^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}},\qquad \sigma^{\mathrm{corr}}_{ij} = \frac{\sigma^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}}. \)

Unique intensities are merged by inverse-variance weighted mean: \( I_h = \frac{\sum_j w_j I^{\mathrm{corr}}_{ij}}{\sum_j w_j},\qquad w_j = \frac{1}{(\sigma^{\mathrm{corr}}_{ij})^2}. \)

An internal-consistency term can inflate uncertainties when multiple observations are present, in the spirit of XSCALE.

10.5 Merging statistics

Per-shell and overall merging statistics are computed on corrected intensities, including:

  • number of observations and of unique reflections, and multiplicity,

  • mean \(I/\sigma(I)\),

  • \(R_\mathrm{meas}\) (the redundancy-independent Diederichs–Karplus form) from within‑HKL deviations,

  • \(\mathrm{CC}_{1/2}\) (half-set correlation) and, when a reference dataset is supplied, \(\mathrm{CC}_\mathrm{ref}\),

  • completeness against the enumerated reflections for the cell and symmetry.

The error model is refined as \(\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2\) with a systematic floor \(\sigma\ge b|I|\); the asymptotic signal-to-noise \(\mathrm{ISa}=1/b\) is reported and written to the output files.

10.6 Rotation datasets: combining partials into fulls (3D integration)

In a rotation scan a reflection is recorded as a series of partials spread across the frames its rocking curve crosses. Merging those partials directly would force the merge error model to absorb the rocking-curve slicing as if it were measurement noise, capping the achievable \(I/\sigma\). For rotation data Jungfraujoch instead combines each reflection’s partials into a single full intensity first, then scales and merges the fulls — a 3D integration over the rocking curve.

The combine groups each reflection’s partials into rocking events (contiguous runs of frames) and reduces each event to one full:

  • De-biased weighted sum. Partials are combined by inverse-variance weighting, where each partial’s variance is its background-noise component plus the model signal shared across the event (Kabsch profile-fit form). Using the shared model signal rather than the individual down-fluctuating intensity stops weak partials from being over-weighted, which would otherwise inflate the merged error model. The weights depend on the full, so the estimate is iterated.

  • Captured fraction. The partiality summed over the event, \(f=\min(1,\sum_j p_j)\), measures how completely the rocking curve was sampled. A full whose curve was captured below a threshold (--min-captured-fraction, default 0.7 for rotation) is dropped — an event seen over only a small fraction of its curve is unreliable however many frames it spans. (The per-partial minimum-partiality cut of §10.2 still applies upstream, in the per-frame scaling.)

  • Capture-aware uncertainty. A full captured incompletely (\(f<1\)) is extrapolated and biased high. The unobserved fraction is charged as an extra systematic uncertainty, \(\sigma^2 \leftarrow \sigma^2 + \big(c\,(1-f)\,I\big)^2\), so the merge down-weights these extrapolated fulls and the error model treats their scatter as expected. It is enabled by default for the rotation path.

The fulls are then re-scaled in the XDS sense — a per-image scale refit directly on the complete reflections under the unity partiality model — and merged (§10.4). Because every merged observation is now a counting-statistics-limited full rather than a partiality-divided slice, the error model reaches a far higher asymptotic \(I/\sigma\).

After scale-fulls, two optional correction surfaces can be fitted on the combined fulls (rotation only, both off by default), each an alternating multiplicative refinement of the per-full scale against the merged reference:

  • Decay (-B). Radiation damage weakens later frames more at higher resolution — a resolution×time (Debye–Waller) systematic the resolution-flat per-image scale cannot capture. A single global relative-\(B\) rate is fitted, \(\ln(I_\mathrm{ref}/I_\mathrm{obs}) = 2\,(\mathrm{d}B/\mathrm{d}n)\,(n-\bar n)\,s^2\) (frame \(n\), \(s^2 = 1/4d^2\)), and folded into the scale. It engages only when the total relative-\(B\) over the run exceeds a physical floor (2 Ų); below that the decay is negligible and “correcting” it only spreads symmetry equivalents (which sit at the same \(s^2\) but different frames).

  • Absorption (--absorption). A smooth multiplicative factor over the diffracted-beam direction expressed in the goniometer (crystal) frame: each full’s predicted detector position gives the lab diffracted direction, de-rotated by the spindle so a fixed crystal-frame direction is sampled at many rotation angles and its grid cell is well-determined. Negligible at hard X-rays / thin crystals; it matters at low photon energy. Its gain is largest on model-based metrics — a smooth absorption error largely cancels among symmetry mates (small effect on the error model / ISa) but still biases the intensities from their true values (a measurable \(R_\mathrm{free}\) improvement).

Both surfaces are cross-validated: fitted on even-numbered frames and kept only if they improve the held-out odd-frame symmetry-equivalent agreement by a clear margin (and vice versa). A surface fitted to noise where its systematic is absent therefore does not generalize and is discarded — an opt-in correction never adds scatter.

11. Mosaicity and “profile radius” monitoring

11.1 Profile radius (intrinsic excitation-error width)

The “profile radius” is the intrinsic angular width of a reflection — crystal mosaicity plus beam divergence — estimated from the spread of \(\Delta_\mathrm{Ewald}\) over indexed spots, \( R \approx \sqrt{\tfrac{1}{N}\sum_i \Delta_{\mathrm{Ewald},i}^2}. \) When the beam has a finite energy bandwidth, that bandwidth smears each reflection radially by \(\sigma_\mathrm{bw}\approx \mathrm{bandwidth}\cdot\lambda/2d^2\) (largest at high resolution), which also broadens the measured \(\Delta_\mathrm{Ewald}\) spread. Since prediction re-applies the bandwidth term per reflection (§8.2), this contribution is deconvolved from the estimate — \(R^2 = \langle\Delta_\mathrm{Ewald}^2\rangle - \langle\sigma_\mathrm{bw}^2\rangle\) — so that \(R\) is the intrinsic width and bandwidth is not double-counted. Still predictions use an excitation-error cutoff proportional to \(R\).

11.2 Mosaicity from rotation data

For rotation data the mosaicity \(\sigma_M\) is estimated by maximum likelihood from the rocking offsets \(\tau\) of indexed spots, using the XDS reflection-fraction model \(R(\tau;\sigma_M/\zeta)\) (Kabsch 2010): each spot’s exact Bragg angle is located near its frame, \(\zeta\) (the rotation-axis Lorentz component) is computed, and \(\sigma_M\) is chosen to maximize \(\sum_i \log R(\tau_i;\sigma_M/\zeta_i)\).

The \(\phi\) search window for the Bragg angle is set wider than the oscillation, so that reflections recorded at large rocking offset are included. These tail reflections carry most of the information about the mosaic width; a window limited to the oscillation range would truncate the \(\tau\) distribution and bias \(\sigma_M\) low.

The estimated mosaicity feeds the rotation prediction (how many frames each reflection spans, §8.3) and the rotation partiality (§10.2). It is held fixed during scaling: in the per-image scale fit the mosaicity is degenerate with the scale \(G\) (both rescale the predicted intensity), so refining it there is unstable. A correct mosaicity matters because it controls both how much of each rocking curve is captured and the partiality used to form fulls (§10.6); too small a value truncates the captured curve and over-peaks the partiality, degrading the combined fulls.

12. Auxiliary statistics: ⟨I/σ(I)⟩ and Wilson plot

12.1 Per-shell ⟨I/σ(I)⟩

For monitoring integration quality, Jungfraujoch reports mean \(\langle I/\sigma(I)\rangle\) in a fixed number of resolution shells. Shelling is performed in \(1/d^2\) space (typical of crystallographic practice).

12.2 Wilson plot (B-factor proxy)

A Wilson-type analysis is computed by binning intensities by resolution and fitting: \( \langle I\rangle \propto \exp\!\left(-\frac{B}{2}\frac{1}{d^2}\right), \) i.e. \( \log \langle I\rangle = \mathrm{const} - \frac{B}{2}\left(\frac{1}{d^2}\right). \) A linear regression of \(\log\langle I\rangle\) vs \(1/d^2\) provides an estimate of \(B\), subject to basic quality checks (e.g. \(R^2\) threshold).

13. Practical notes and limitations

  • Bragg integration is profile-fitted by default (per-shell Gaussian profile, Kabsch extraction; §9.3), with plain box summation available as a fallback (--integrator boxsum). The profiles are built per frame from that frame’s strong spots, which suits fast-feedback and serial/streaming use; a profile shared across many frames (as in full offline workflows) is not currently formed.

  • Space-group symmetry beyond centering absences is not necessarily enforced during prediction/integration unless the space group is supplied and used downstream.

  • Resolution masking and ice rings are controllable; including ice-ring spots in indexing can improve robustness for some samples but may bias refinement in others.

  • Rotation vs still modes differ substantially in prediction and scaling: partiality is angle-driven in rotation data, while stills are predicted (within an excitation-error window) and scaled with unit partiality.

  • Space-group determination. When no space group is supplied, a POINTLESS-like search scores Laue-group symmetry (CC of \(I(h)\) vs \(I(Rh)\) plus merge self-consistency) and detects screw/centering absences from the \(P1\)-merged intensities. The self-consistency test is calibrated so a merohedral twin — whose twin law forces non-equivalent reflections together and inflates the merged \(\chi^2\) — stays in its true lower symmetry rather than being over-promoted to the holohedral group.

  • Twinning check. A Padilla–Yeates \(L\)-test (\(\langle|L|\rangle\), \(\langle L^2\rangle\)) and the second moment \(\langle I^2\rangle/\langle I\rangle^2\) (taken per resolution shell with noise-only shells skipped and Wilson outliers rejected, so a single strong reflection in a collapsed-mean shell cannot skew it) are written to the merged mmCIF as a twinning diagnostic. Twinning is only flagged in Laue classes where a merohedral twin law can exist; the holohedral high-symmetry classes (\(4/mmm\), \(6/mmm\), \(m\bar{3}m\), and \(\bar{3}m\) on a rhombohedral lattice) are exempt, so a low \(\langle|L|\rangle\) there is reported as a statistical artefact rather than twinning.

  • Outlier rejection. Merging applies an optional per-observation median-based \(N\sigma\) cut (default 6σ for rot3d) and an optional per-crystal \(\Delta\mathrm{CC}_{1/2}\) image rejection (--reject-delta-cchalf, CrystFEL-style, off by default). The same \(N\sigma\) cut is fed back into the error model: after an initial \(a,b\) fit the parameters are re-fit once on the reflections that survive rejection (dropping any whose squared deviation exceeds \(N\sigma^2\,[a\,\sigma^2 + (b\,\langle I\rangle)^2]\)), so the calibrated errors describe the reflections that actually enter the merge rather than the pre-rejection pool.

  • Automatic resolution cutoff. By default the reported/written high-resolution limit is trimmed where \(\mathrm{CC}_{1/2}\) falls off (logistic, target 0.30); --scaling-high-resolution overrides it and --resolution-cutoff off disables it.

  • Intensities only. The merged output carries intensities (mmCIF intensity_meas, MTZ IMEAN/SIGIMEAN); it does not convert to amplitudes \(|F|\) (no French–Wilson / truncate step) — do that downstream.

\ No newline at end of file diff --git a/DEPLOYMENT.html b/DEPLOYMENT.html new file mode 100644 index 00000000..979f4ea7 --- /dev/null +++ b/DEPLOYMENT.html @@ -0,0 +1,38 @@ + Deployment — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Deployment

To deploy Jungfraujoch, one needs to follow four steps:

  1. Install main Jungfraujoch code and frontend web interface

  2. Flash the U55C FPGA card with a proper image and install Linux kernel driver

  3. Install Jungfraujoch writer

  4. Install Python OpenAPI client

Installation procedure depend a lot on the operating system. For RedHat Enterprise Linux 8, Rocky 8, or compatible installation can be done with prebuilt RPMs and is relatively straightforward. For other systems one needs to build software from source. Both ways will be presented.

Install main Jungfraujoch code and frontend web interface

On RHEL 8 systems there is a jfjoch-<version>-1.el8.x86_64.rpm that needs to be installed and contains all the necessary software and web interface.

On other OSes one needs to compile Jungfraujoch from source (from the repo directory):

$ mkdir build
+$ cd build
+$ cmake .. -DCMAKE_INSTALL_PREFIX=<directory to install>
+$ make
+$ sudo make install  
+

For manual installation, we recommend to use non-standard directory (like /opt/jfjoch), to facilitate upgrades and removal. For DKMS to manage kernel module sources it is necessary to copy driver sources to /usr/src/jfjoch-<VERSION> directory. This requires extra flag in cmake -DJFJOCH_INSTALL_DRIVER_SOURCE=ON.

Frontend web user interface has to be built separately with:

$ cd build
+$ make frontend
+

Frontend files (.html and .js) will be placed in frontend/dist (outside of build/ directory!) and has to be copied to a general location, e.g. /usr/local/jfjoch/frontend or /opt/jfjoch/frotend.

Flash the U55C FPGA card with a proper image and install Linux kernel driver.

Firmware flashing

  1. Check that the card is detected by OS with “lspci |grep Xilinx” and check the PCIe bus/device/function (BDF) number, 11:00.0 in this case:

$ lspci |grep Xilinx
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2)
+

Note the device number 3450 that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and rev 2 identifying release of the firmware.

  1. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given):

$ sudo lspci -vv -s <PCIe slot number>
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450
+(...)
+LnkSta:     Speed 16GT/s (ok), Width x8 (ok)
+(...)
+

  1. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn’t allways reach correct timing).

  2. Flash the card with xbflash.qspi tool (part of Jungfraujoch). For fresh card use:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above> --bar-offset 0x1f06000 
+

For card that was already flashed with Jungfraujoch images:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above>
+

It is necessary to confirm the operation by pressing Y key or one can add --force option to avoid confirmation. It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions.

  1. Cold reboot:

sudo ipmitool chassis power cycle
+

Install PCIe driver

For first run it is though recommended to try the driver without installing to the kernel directory:

$ cd fpga/pcie_driver
+$ make
+$ sudo insmod jfjoch.ko
+

Check with dmesg that the device was properly found:

$ dmesg |grep jfjoch
+[  431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142)
+[  431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a
+

If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. On RHEL 8 you can install prebuilt RPM provided in the Gitlab package registry. On other systems follow procedure in PCIe driver.

NOTE: Driver installation procedure on non-RHEL 8 systems is not well understood/optimized at the moment.

NOTE: In case driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk if driver is updated:

$ sudo dracut -f
+

Configure network

Configure switch according to FPGA network guide - specifically set manual speed and turn off auto-negotiation for the port used to connect U55C card and connect card to switch.

Running Jungfraujoch software

Main Jungfraujoch service is called jfjoch_broker. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. It is recommended to run the service as systemd service.

jfjoch_broker takes two parameters: JSON configuration file and HTTP port (default is 5232). Example JSON files are placed in etc/ folder. JSON file format is also explained in the OpenAPI definition, as jfjoch_settings data structure.

When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring.

Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one jfjoch_broker on a single machine, each confined to a disjoint subset of GPUs, set CUDA_VISIBLE_DEVICES; setting CUDA_DEVICE_ORDER=PCI_BUS_ID keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host:

CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232
+CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233
+

To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. These need to be obtained from the PSI Detector Group.

Card verification

To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use jfjoch_fpga_test tool. For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs:

jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000
+

Or 1M pixel system with one FPGA card:

jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000
+

Install Jungfraujoch writer

Jungfraujoch writer is an additional service, that can connect to jfjoch_broker ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard.

At the moment it is better to have a separate machine, with access to distributed file system, for writing images.

Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands:

mkdir build
+cd build
+cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch
+

Install Jungfraujoch image viewer

Jungfraujoch viewer is X-ray diffraction image viewer, that is optimized to open Jungfraujoch HDF5 files.

The viewer is a Qt application and it requires recent version of the library, therefore it is an optional dependency.

To include it in the building of Jungfraujoch use -DJFJOCH_VIEWER_BUILD=ON directive for CMake:

mkdir build
+cd build
+cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch
+

Install Jungfraujoch Python client

Use pip:

pip install jfjoch-client
+
\ No newline at end of file diff --git a/DETECTORS.html b/DETECTORS.html new file mode 100644 index 00000000..204ba0a9 --- /dev/null +++ b/DETECTORS.html @@ -0,0 +1 @@ + Supported detectors — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Supported detectors

PSI detectors

Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfruajoch controls the detector via statically compiled slsDetectorPackage into its source code. It is important that detector firmware has to match slsDetectorPackage version used in Jungfraujoch (8.0.2 at the moment). See PSI Detector group website for details.

DECTRIS detectors

Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles output data stream of the DCU. This mode, called “lite” mode, doesn’t use FPGA boards, but mostly CPUs and GPUs for indexing. The mode is currently experimental and intended for low data rates (100 Hz).

\ No newline at end of file diff --git a/DETECTOR_GEOMETRY.html b/DETECTOR_GEOMETRY.html new file mode 100644 index 00000000..e75d8354 --- /dev/null +++ b/DETECTOR_GEOMETRY.html @@ -0,0 +1 @@ + Detector geometry — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Detector geometry

At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location vs. detector frame. It is not recommended to place detector modules stacked.

The simplest case is detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance and wavelength.

For more complex case, one can provide tilt of the detector rotation in PyFAI convention. This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector is defined with three rotation angles: rot1 (rotating detector right), rot2 (rotating detector downwards), rot3 (rotating detector clockwise). See PyFAI documentation for more details.

Macromolecular crystallography convention for the vertical direction

One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, with Y values increasing downwards. This is also consistent with computer image formats.

However, other techniques (as well as internal operation of PSI X-ray detectors) might follow convention, for point (0,0) being in the bottom left corner and Y values increasing upwards. Such a convention is used, for example, by PyFAI.

In general, convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows mirroring detector in Y.

Extra care has to be taken by the user to ensure that no errors are made.

\ No newline at end of file diff --git a/FPGA.html b/FPGA.html new file mode 100644 index 00000000..a9bbcc8f --- /dev/null +++ b/FPGA.html @@ -0,0 +1,16 @@ + FPGA smartNIC — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA smartNIC

See separate document for installation instructions.

Hardware

Currently supported FPGA is only Xilinx Alveo U55C.

See AMD/Xilinx webpage for card user guide (UG1469). According to the user guide:

Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling.
+

Card needs to be placed in PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C.

Two variants of the card are available:

  • 100g - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch.

  • 8x10g - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

See network documentation for details of network.

Building firmware

Xilinx Vivado version has to precisely match version described in [the system requirements](../README.md. only when vivado and vitis_hls are detected in the path.

Xilinx Vivado

The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version 2022.2 installed on the machine. Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - specifically newer versions of Vivado will not work.

In additional to Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary:

  • Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see Xilinx website, to build 100g design.

  • Paid 10G/25G Subsystem for Ultrascale+ to build 8x10g design. PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams generated by PSI continuous integration pipeline for 8x10g is only allowed for non-commercial use.

HLS compilation

Make HLS routines:

mkdir build
+cd build
+cmake ..
+make hls
+

Synthesis

Create PCIe 100g bitstream with the following command:

mkdir build
+cd build
+cmake ..
+make pcie_100g
+

and 8x10g:

mkdir build
+cd build
+cmake ..
+make pcie_8x10g
+

When Vivado is not present

During CMake execution, the following executables: vivado and vitis_hls must be present in the path. If not, build targets will not be generated, and such or similar error message will show up:

$ make pcie_100g
+make: *** No rule to make target 'pcie_100g'.  Stop.
+

Gitlab CI

If Gitlab CI is properly set-up, firmware will be automatically built for every commit that starts with FPGA. Built firmware should be downloaded as MCS files.

Frame generator

Jungfraujoch card is equipped with frame generator. It allows to simulate JUNGFRAU detector without having access to such system. It is placed in parallel to Ethernet MAC - so it is placed before the network stack and before any processing happening on the card. In the future a redirection will be possible to send the simulated stream through the 100G TX network link. Frame generator is written in HLS and controlled with AXI-Lite.

\ No newline at end of file diff --git a/FPGA_DATA_ANALYSIS.html b/FPGA_DATA_ANALYSIS.html new file mode 100644 index 00000000..458ec865 --- /dev/null +++ b/FPGA_DATA_ANALYSIS.html @@ -0,0 +1 @@ + FPGA data analysis — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA data analysis

Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities.

Pixel mask

Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) and will be excluded from a subsequent analysis.

ADU histogram

Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature of unconverted values. This is done on a module-basis and works with bins with 32 ADU width.

For EIGER this can be used as just a histogram procedure.

JUNGFRAU conversion

For JUNGFRAU module images are converted from ADUs to energy value and divided by a given number to keV units. Result of the operation is rounded to integers.

Pixel thresholding

Pixel range can be specified. Pixels below a minimum threshold will be assigned zero. Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled.

Frame summation

Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together.

Azimuthal integration

To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. Corrections were implemented according to formulas developed by Jensen et al. (J. Synchr. Rad., 29, 1420-1428, 2022).

Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited as 1024 per detector module. This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit pixed point integer implementation and is tuned for standard SAXS/WAXS range.

As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows.

Spot finding

Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels:

  1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run).

  2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding,

  3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation,

  4. Pixel value - pixels above certain threshold value can be marked as strong,

  5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong,

  6. Connected pixels - strong pixels can be discarded if they are “alone”, so their 8 directly neighboring pixels are not counted as strong pixels.

While besides bad pixels criterion, all the above are optional (can be turned off), only pixels that fulfill all enabled criteria are selected as strong pixels.

SNR ratio calculation

Signal-to-noise ratio is calculated for a rectangular area. In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. This is dictated by the data flow within the FPGA. In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. Given very large box size, approximation are made, for example that N ≈ N-1 in calculating standard deviation.

Region-of-interest (ROI) integration

Each pixel in a module can be assigned to one of 64 ROIs. For each ROIs, sum, sum of squares, max count, and number of valid pixels will be calculated. Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis.

ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal (Q-range with an optional φ-sector) — are also evaluated on CPU by the shared image_analysis/roi/ engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path.

Pixel statisitics

The following statistics are collected for each module:

  • Number of masked pixels

  • Number of saturated pixels (excl.masked)

  • Number of error pixels (excl. masked)

  • Sum of valid pixels in the module

  • Minimum value of valid pixels in the module

  • Maximum value of valid pixels in the module Valid pixels are not masked, not saturated, not error pixels.

Square root compression

Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. Scheme was described in Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020. Pixel value X is replaced with sqrt(N*X), where N is integer constant in range 1 to 16.

\ No newline at end of file diff --git a/FPGA_DESIGN.html b/FPGA_DESIGN.html new file mode 100644 index 00000000..75021342 --- /dev/null +++ b/FPGA_DESIGN.html @@ -0,0 +1 @@ + FPGA data flow — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA data flow

The following steps are performed on FPGA (in the order of operation):

  1. UDP header decoding

  2. SLS detector header decoding

  3. State machine that controls data acquisition (start/stop/cancel)

  4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules

  5. ADU histogram for JUNGFRAU

  6. Mask pixels from missing packets with special value

  7. Reorder lines for EIGER to form a proper module

  8. Mask pixels based on provided pixel mask

  9. JUNGFRAU conversion with gain and pedestal corrections

  10. Threshold to zero pixels below certain count value

  11. Integration according to predefined map (e.g., 1D azimuthal integration)

  12. Spot finding

  13. ROI calculation

  14. Image lossy compression using N*sqrt(pixel) values

  15. Send images, analysis results and metadata to host memory via PCI Express

Each step has dedicated core, written in the high-level synthesis. Exact operation of cores for data analysis is explained in dedicated document.

\ No newline at end of file diff --git a/FPGA_LICENSE.html b/FPGA_LICENSE.html new file mode 100644 index 00000000..48d224c5 --- /dev/null +++ b/FPGA_LICENSE.html @@ -0,0 +1,7 @@ + FPGA license — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA license

FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. The license is equivalent of GNU Public License with adaptations for hardware. See OHL webpage for details and FAQs.

CERN Open Hardware Licence Version 2 - Strongly Reciprocal

Preamble

CERN has developed this licence to promote collaboration among hardware designers and to provide a legal tool which supports the freedom to use, study, modify, share and distribute hardware designs and products based on those designs. Version 2 of the CERN Open Hardware Licence comes in three variants: CERN-OHL-P (permissive); and two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this licence, CERN-OHL-S (strongly reciprocal).

The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in unmodified form only.

Use of this Licence does not imply any endorsement by CERN of any Licensor or their designs nor does it imply any involvement by CERN in their development.

1 Definitions

1.1 ‘Licence’ means this CERN-OHL-S.

1.2 ‘Compatible Licence’ means

a) any earlier version of the CERN Open Hardware licence, or

b) any version of the CERN-OHL-S, or

c) any licence which permits You to treat the Source to which it applies as licensed under CERN-OHL-S provided that on Conveyance of any such Source, or any associated Product You treat the Source in question as being licensed under CERN-OHL-S.

1.3 ‘Source’ means information such as design materials or digital code which can be applied to Make or test a Product or to prepare a Product for use, Conveyance or sale, regardless of its medium or how it is expressed. It may include Notices.

1.4 ‘Covered Source’ means Source that is explicitly made available under this Licence.

1.5 ‘Product’ means any device, component, work or physical object, whether in finished or intermediate form, arising from the use, application or processing of Covered Source.

1.6 ‘Make’ means to create or configure something, whether by manufacture, assembly, compiling, loading or applying Covered Source or another Product or otherwise.

1.7 ‘Available Component’ means any part, sub-assembly, library or code which:

a) is licensed to You as Complete Source under a Compatible Licence; or

b) is available, at the time a Product or the Source containing it is first Conveyed, to You and any other prospective licensees

i) as a physical part with sufficient rights and information (including any configuration and programming files and information about its characteristics and interfaces) to enable it either to be Made itself, or to be sourced and used to Make the Product; or ii) as part of the normal distribution of a tool used to design or Make the Product.

1.8 ‘Complete Source’ means the set of all Source necessary to Make a Product, in the preferred form for making modifications, including necessary installation and interfacing information both for the Product, and for any included Available Components. If the format is proprietary, it must also be made available in a format (if the proprietary tool can create it) which is viewable with a tool available to potential licensees and licensed under a licence approved by the Free Software Foundation or the Open Source Initiative. Complete Source need not include the Source of any Available Component, provided that You include in the Complete Source sufficient information to enable a recipient to Make or source and use the Available Component to Make the Product.

1.9 ‘Source Location’ means a location where a Licensor has placed Covered Source, and which that Licensor reasonably believes will remain easily accessible for at least three years for anyone to obtain a digital copy.

1.10 ‘Notice’ means copyright, acknowledgement and trademark notices, Source Location references, modification notices (subsection 3.3(b)) and all notices that refer to this Licence and to the disclaimer of warranties that are included in the Covered Source.

1.11 ‘Licensee’ or ‘You’ means any person exercising rights under this Licence.

1.12 ‘Licensor’ means a natural or legal person who creates or modifies Covered Source. A person may be a Licensee and a Licensor at the same time.

1.13 ‘Convey’ means to communicate to the public or distribute.

2 Applicability

2.1 This Licence governs the use, copying, modification, Conveying of Covered Source and Products, and the Making of Products. By exercising any right granted under this Licence, You irrevocably accept these terms and conditions.

2.2 This Licence is granted by the Licensor directly to You, and shall apply worldwide and without limitation in time.

2.3 You shall not attempt to restrict by contract or otherwise the rights granted under this Licence to other Licensees.

2.4 This Licence is not intended to restrict fair use, fair dealing, or any other similar right.

3 Copying, Modifying and Conveying Covered Source

3.1 You may copy and Convey verbatim copies of Covered Source, in any medium, provided You retain all Notices.

3.2 You may modify Covered Source, other than Notices, provided that You irrevocably undertake to make that modified Covered Source available from a Source Location should You Convey a Product in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source. In each case subsection 3.3 shall apply.

  You may only delete Notices if they are no longer applicable to
+  the corresponding Covered Source as modified by You and You may
+  add additional Notices applicable to Your modifications.
+  Including Covered Source in a larger work is modifying the
+  Covered Source, and the larger work becomes modified Covered
+  Source.
+

3.3 You may Convey modified Covered Source (with the effect that You shall also become a Licensor) provided that You:

a) retain Notices as required in subsection 3.2;

b) add a Notice to the modified Covered Source stating that You have modified it, with the date and brief description of how You have modified it;

c) add a Source Location Notice for the modified Covered Source if You Convey in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source; and

d) license the modified Covered Source under the terms and conditions of this Licence (or, as set out in subsection 8.3, a later version, if permitted by the licence of the original Covered Source). Such modified Covered Source must be licensed as a whole, but excluding Available Components contained in it, which remain licensed under their own applicable licences.

4 Making and Conveying Products

You may Make Products, and/or Convey them, provided that You either provide each recipient with a copy of the Complete Source or ensure that each recipient is notified of the Source Location of the Complete Source. That Complete Source is Covered Source, and You must accordingly satisfy Your obligations set out in subsection 3.3. If specified in a Notice, the Product must visibly and securely display the Source Location on it or its packaging or documentation in the manner specified in that Notice.

5 Research and Development

You may Convey Covered Source, modified Covered Source or Products to a legal entity carrying out development, testing or quality assurance work on Your behalf provided that the work is performed on terms which prevent the entity from both using the Source or Products for its own internal purposes and Conveying the Source or Products or any modifications to them to any person other than You. Any modifications made by the entity shall be deemed to be made by You pursuant to subsection 3.2.

6 DISCLAIMER AND LIABILITY

6.1 DISCLAIMER OF WARRANTY – The Covered Source and any Products are provided ‘as is’ and any express or implied warranties, including, but not limited to, implied warranties of merchantability, of satisfactory quality, non-infringement of third party rights, and fitness for a particular purpose or use are disclaimed in respect of any Source or Product to the maximum extent permitted by law. The Licensor makes no representation that any Source or Product does not or will not infringe any patent, copyright, trade secret or other proprietary right. The entire risk as to the use, quality, and performance of any Source or Product shall be with You and not the Licensor. This disclaimer of warranty is an essential part of this Licence and a condition for the grant of any rights granted under this Licence.

6.2 EXCLUSION AND LIMITATION OF LIABILITY – The Licensor shall, to the maximum extent permitted by law, have no liability for direct, indirect, special, incidental, consequential, exemplary, punitive or other damages of any character including, without limitation, procurement of substitute goods or services, loss of use, data or profits, or business interruption, however caused and on any theory of contract, warranty, tort (including negligence), product liability or otherwise, arising in any way in relation to the Covered Source, modified Covered Source and/or the Making or Conveyance of a Product, even if advised of the possibility of such damages, and You shall hold the Licensor(s) free and harmless from any liability, costs, damages, fees and expenses, including claims by third parties, in relation to such use.

7 Patents

7.1 Subject to the terms and conditions of this Licence, each Licensor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in subsections 7.2 and 8.4) patent licence to Make, have Made, use, offer to sell, sell, import, and otherwise transfer the Covered Source and Products, where such licence applies only to those patent claims licensable by such Licensor that are necessarily infringed by exercising rights under the Covered Source as Conveyed by that Licensor.

7.2 If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Covered Source or a Product constitutes direct or contributory patent infringement, or You seek any declaration that a patent licensed to You under this Licence is invalid or unenforceable then any rights granted to You under this Licence shall terminate as of the date such process is initiated.

8 General

8.1 If any provisions of this Licence are or subsequently become invalid or unenforceable for any reason, the remaining provisions shall remain effective.

8.2 You shall not use any of the name (including acronyms and abbreviations), image, or logo by which the Licensor or CERN is known, except where needed to comply with section 3, or where the use is otherwise allowed by law. Any such permitted use shall be factual and shall not be made so as to suggest any kind of endorsement or implication of involvement by the Licensor or its personnel.

8.3 CERN may publish updated versions and variants of this Licence which it considers to be in the spirit of this version, but may differ in detail to address new problems or concerns. New versions will be published with a unique version number and a variant identifier specifying the variant. If the Licensor has specified that a given variant applies to the Covered Source without specifying a version, You may treat that Covered Source as being released under any version of the CERN-OHL with that variant. If no variant is specified, the Covered Source shall be treated as being released under CERN-OHL-S. The Licensor may also specify that the Covered Source is subject to a specific version of the CERN-OHL or any later version in which case You may apply this or any later version of CERN-OHL with the same variant identifier published by CERN.

8.4 This Licence shall terminate with immediate effect if You fail to comply with any of its terms and conditions.

8.5 However, if You cease all breaches of this Licence, then Your Licence from any Licensor is reinstated unless such Licensor has terminated this Licence by giving You, while You remain in breach, a notice specifying the breach and requiring You to cure it within 30 days, and You have failed to come into compliance in all material respects by the end of the 30 day period. Should You repeat the breach after receipt of a cure notice and subsequent reinstatement, this Licence will terminate immediately and permanently. Section 6 shall continue to apply after any termination.

8.6 This Licence shall not be enforceable except by a Licensor acting as such, and third party beneficiary rights are specifically excluded.

\ No newline at end of file diff --git a/FPGA_NETWORK.html b/FPGA_NETWORK.html new file mode 100644 index 00000000..a349acd8 --- /dev/null +++ b/FPGA_NETWORK.html @@ -0,0 +1 @@ + FPGA network — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA network

The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 is lower port (when PCIe connector is on the bottom). The card FPGA design is offered in two variants 100g and 8x10g. These have different behavior regarding the network:

100g this variant operates QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a switch. QSFP28 transceivers are necessary.

8x10g this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

Transceivers

AMD doesn’t provide transceiver compatibility matrix for Alveo U55C. In our experience operating the card we haven’t seen issues with transceivers from various providers (FS.com, Mellanox, Finnisar). We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn’t support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven’t encountered such a situation.

Switch configuration

Special care has to be taken for switch operation, given the FPGA core doesn’t support auto-negotiation. It is necessary to configure switch port to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000).

Network LEDs

Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs means all ports connected, orange LEDs at least one of the ports connected.

Network stack

Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed.

The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. The card won’t respond to broadcast ICMP pings.

Each link can be put in direct mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. This settings should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any 10 Gbit/s link on the same card. Currently direct mode is turned OFF for 100g design and ON for 8x10g design. This can be manually adjusted for each link.

\ No newline at end of file diff --git a/FPGA_PCIE_DRIVER.html b/FPGA_PCIE_DRIVER.html new file mode 100644 index 00000000..99ecdedc --- /dev/null +++ b/FPGA_PCIE_DRIVER.html @@ -0,0 +1,10 @@ + FPGA PCIe driver — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA PCIe driver

Compilation

To compile kernel module type:

make
+

Installation

To install kernel module, you need to have root permissions and run:

sudo make install
+

Loading driver into kernel

After installing the kernel driver, it should be possible to insert it into the kernel via:

modprobe jfjoch
+

Ownership of the character devices

By default, character devices /dev/jfjoch<device number> are owned by root (user/group) and are not accessible by others. This means that jfjoch_broker must be running as superuser, which might not be optimal for security reasons in most cases. The behavior can be changed by creating udev rules. Create a file called /etc/udev/rules.d/99-jfjoch.rules with the following content:

KERNEL=="jfjoch*" OWNER="<UNIX username>" GROUP="<UNIX group>"
+

It is OK to provide only group, for example to make the devices accessible by group jungfrau:

KERNEL=="jfjoch*" GROUP="jungfrau"
+

DKMS

To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. For other systems, it is necessary to follow the procedure below, though it is not well tested.

This first requires to install DKMS - for RHEL it is available via EPEL repository:

sudo dnf install dkms
+

Then use script provided in the driver directory to copy driver code to DKMS directory:

./install_dkms.sh
+

If upgrading the driver, please first remove current driver from DKMS system:

dkms remove jfjoch -v <version> --all
+

Driver parameters

Currently, there is one driver parameter nbuffers, that defines count of exchange buffers (see below). This can be adjusted in the modprobe operation, for example:

modprobe jfjoch nbuffers=1024
+

Exchange buffers

The parameter defines number of buffers used to exchange data between card and host application. Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. These buffers are used by both card-to-host and host-to-card operations.

Buffers use special allocation, as they are continuous in physical address space, which helps the FPGA card to transfer all data associated with detector module in two DMA transfers (one data, one metadata). Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in power of two, 4 MiB is safe number for one buffer size. Buffer can be mapped into user space, but performing mmap system call on the /dev/jfjoch<number of device> character device.

Buffer count can be adjusted by setting nbuffers parameter. There are two considerations for setting optimal value:

  1. For card-to-host transfers, minimal value is roughly <number of threads in receiver> * <number of modules processed by thread; usually equal to number of modules per card>, this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64.

  2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. So minimal count is <number of modules> * (3 + 3 * <number of storage cells>).

Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration.

Known problems

To avoid inconsistent behavior, this driver won’t load if release number differs between the kernel driver and FPGA card.

CMake file

While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. It is not made for actual compilation of the kernel driver and should not be used for that purpose.

Character device access

For each FPGA device a character device is created called /dev/jfjoch<number of device>. When device is opened two operations are possible: mmap() to map exchange buffers ioctl() to communicate with the cards Interfacing should be done through the JungfraujochDevice class in fpga/host_library directory.

Sysfs access

Certain performance counters can be read through sysfs mechanism in the kernel. One needs to cat files in /sys/class/misc/jfjoch<number of device>/ directory.

RHEL 9.5+ issue

RedHat Enterprise Linux 9.5 backported modification to settings virtual memory flags from Linux kernel 6.3, while still operating kernel version 5.14. It is complicated to come up with a single rule to select when newer functions should be used, so it works with RHEL 9.5+, while still being compatible with other Linux distributions. It is even more complex given not all RHEL compatible distributions adopted the change at the same version. For the moment the quick fix is to define an environment variable HAVE_VM_FLAGS_SET before making the kernel.

\ No newline at end of file diff --git a/FPGA_SETTINGS.html b/FPGA_SETTINGS.html new file mode 100644 index 00000000..1f33d687 --- /dev/null +++ b/FPGA_SETTINGS.html @@ -0,0 +1 @@ + FPGA advanced reference — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

FPGA advanced reference

Register map

FPGA setup can be done via registers:

Address

Bits

Meaning

Mode

Notes

0x000000 - 0x00FFFF

Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory)

0x010000

32

Action Control Register

Bit 0 - Action start

R/W

Bit 1 - Action idle

R

Bit 2 - Action cancel

R/W

cleared on reset or action start

Bit 3 - Clear network counters

R/W

cleared on reset

Bit 12:4 - Debug signals (see action_config.v for details)

R

Bit 16 - AXI Mailbox interrupt 0

R

0x010004

32

Reserved

-

0x010008

32

Reserved

-

0x01000C

32

GIT SHA1

R

0x010010

32

Reserved

R

0x010014

32

Reserved

R

0x010018

32

Jungfraujoch FPGA variant

R

0x01001C

32

Reserved

R

0x010020

32

Max. number supported detector modules

R

constant

0x010024

32

Reserved

R

constant

0x010028

64

Pipeline stalls before writing to host memory

R

reset on action start

0x010030

64

Pipeline stalls before accessing HBM

R

reset on action start

0x010038

32

FIFO status (see action_config.v for details)

R

0x01003C

32

Size of single HBM channel in bytes (default value for the particular card)

R/W

should not be altered for standard operation

0x010040

64

Packets processed by the action

R

cleared on reset or action start

0x010048

64

Valid ethernet packets

R

cleared on reset

0x010050

64

Valid ICMP packets

R

cleared on reset

0x010058

64

Valid UDP packets

R

cleared on reset

0x010060

64

Valid detector packets processed by the card

R

cleared on reset

0x010068

64

Packets flagged as errors by CMAC

R

cleared on reset

0x010070

64

Pipeline stalls before data processing

R

reset on action start

0x010078

64

AXI-beats before accessing HBM

R

reset on action start

0x010080

64

AXI-beats before data processing

R

reset on action start

0x010088

64

AXI-beats before host writer

R

reset on action start

0x010090

64

Last encountered SwissFEL pulse ID

R

cleared on reset

0x010100

32

Spot finder photon count threshold

R/W

0x010104

32

Spot finder signal-to-noise ratio threshold (single-precision float)

R/W

0x010200

64

MAC address source for internal frame generator

R/W

network byte order

0x010208

32

IPv4 address source for internal frame generator

R/W

network byte order

0x01020C

32

Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.)

R/W

0x010210

32

Data collection mode

R/W

Bit 0 - Conversion to photons

Bit 1 - Output extend to 32-bit

Bit 2 - Output is unsigned integer

Bit 3 - Use sq. root lossy compression

Bit 7 - JUNGFRAU fixed G1 mode

Bit 8 - Set to zero values below threshold

Bit 16:31 - Data collection ID (carried with completions)

0x010214

32

Photon energy in keV (single-precision float)

R/W

0x010218

32

Number of frames expected in the data collection (defines termination condition)

R/W

0x01021C

32

Number of storage cells

R/W

0x010220

32

Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.)

R/W

0x010224

32

Coefficient for sq. root compression (need to set bit in data collection mode to apply)

R/W

0x010225

32

Threshold; set values below set to zero (need to set bit in data collection mode to apply)

R/W

0x030000 - 0x03FFFF

AXI Mailbox for Work Request / Work Completion

See Xilinx PG114 for register map

0x040000 - 0x04FFFF

QuadSPI flash

See Xilinx PG153 for register map

0x050000 - 0x05FFFF

Interrupt controller

See Xilinx PG099 for register map

0x060000 - 0x06FFFF

Load calibration (HLS)

0x070000 - 0x07FFFF

AXI Firewall

See Xilinx PG293 for register map

0x080000 - 0x08FFFF

Frame generator (HLS)

0x090000 - 0x09FFFF

PCIe DMA control

See Xilinx PG195 for register map

0x0A0000 - 0x0AFFFF

I2C clock generator

See Xilinx PG195 for register map

0x0C0000 - 0x0FFFFF

Xilinx Card Management Solution Subsystem management subsystem

See Xilinx PG348 for register map

0x100000 - 0x10FFFF

MAC 10G / CMAC 100G

See Xilinx PG210/PG203 for register map

0x110000 - 0x11FFFF

MAC 10G

See Xilinx PG210 for register map

0x120000 - 0x12FFFF

MAC 10G

See Xilinx PG210 for register map

0x130000 - 0x13FFFF

MAC 10G

See Xilinx PG210 for register map

0x140000 - 0x14FFFF

MAC 10G

See Xilinx PG210 for register map

0x150000 - 0x15FFFF

MAC 10G

See Xilinx PG210 for register map

0x160000 - 0x16FFFF

MAC 10G

See Xilinx PG210 for register map

0x170000 - 0x17FFFF

MAC 10G

See Xilinx PG210 for register map

0x200000 - 0x20FFFF

Eth/IPv4 network stack for interface #0

0x210000 - 0x21FFFF

Eth/IPv4 network stack for interface #1

0x220000 - 0x22FFFF

Eth/IPv4 network stack for interface #2

0x230000 - 0x23FFFF

Eth/IPv4 network stack for interface #3

0x240000 - 0x24FFFF

Eth/IPv4 network stack for interface #4

0x250000 - 0x25FFFF

Eth/IPv4 network stack for interface #5

0x260000 - 0x26FFFF

Eth/IPv4 network stack for interface #6

0x270000 - 0x27FFFF

Eth/IPv4 network stack for interface #7

0x400000 - 0x47FFFF

64

Address table: decodes handles used by load_calibration and host_writer to DMA addresses

AXI Mailbox

AXI mailbox is used to send work request from host to action, and receive work completions. Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114).

Work request has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Work completion has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Special values:

65534 - start of data collection

65535 - end of data collection

15

31

Data collection ID

HBM memory

Interface number

Core

Meaning

0-1

jf_conversion

Gain factor G0

2-3

jf_conversion

Gain factor G1

4-5

jf_conversion

Gain factor G2

6-7

jf_conversion

Pedestal G0

8-9

jf_conversion

Pedestal G1

10-11

jf_conversion

Pedestal G2

12-13

integration

Integration map

14-15

integration

Integration weights

16-17

spot_finder_mask

Spot finder resolution

18-19

roi_calc

ROI calculation

20-21

frame_generator

Frame generator

22-27

load_from_hbm

Frame summation

\ No newline at end of file diff --git a/HARDWARE.html b/HARDWARE.html new file mode 100644 index 00000000..e1cec205 --- /dev/null +++ b/HARDWARE.html @@ -0,0 +1 @@ + Hardware requirements — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Hardware requirements

Operating Jungfraujoch requires the following:

  1. High performance server

  2. FPGA board(s) installed in the server

  3. (optionally) GPU boards

  4. (optionally) 100G switch to connect FPGA and the detector

Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include AMD FPGA boards. Therefore, the two has to be purchases separately. This might have impact on the warranty for the hardware and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn’t take any responsibility for warranty limitations for operating FPGA boards in the server. Having said this - we didn’t encounter any hardware issues so far.

High performance server

PSI is using HPE DL380 Gen11 servers are the moment to operate Jungfraujoch systems. However, this is because of general preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system from any other vendor with similar specification should work as well.

At PSI, we use the following configuration of HPE DL380 Gen11 to operate 9M pixel detectors at 2 kHz is as follows:

  • 2 x Intel Xeon 8558P

  • 512 GB RAM

  • 2 x Nvidia L4 GPU (for indexing)

  • 1 x Nvidia Connect-X 6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs)

  • Copper 1G/10G network

PCI slots

When ordering the system it is important to ensure enough PCIe cards can be accommodated in the system. In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2x GPU, 1x network

Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK.

FPGA

Jungfraujoch is built for AMD/Xilinx U55C (A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported.

Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires using TX delay functionality in the detector.

GPUs

Operating fast-feedback indexer code requires operation of a graphic processing unit from Nvidia. For practical reasons, i.e. power consumption and cost, we choose inference grade card Nvidia L4. In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work.

Network switch

Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs 8x10g variant of the Jungfraujoch FPGA image, which allows to directly connect 4 JUNGFRAU modules to one U55C card.

Such configuration is however impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujochs server. In this case one needs a network switch.

We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. For switches with only 100G ports it is important to ensure, that these can be split into 4x10G ports to connect the detector.

\ No newline at end of file diff --git a/HDF5.html b/HDF5.html new file mode 100644 index 00000000..d9345352 --- /dev/null +++ b/HDF5.html @@ -0,0 +1,18 @@ + HDF5 / NeXus data format — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

HDF5 / NeXus data format

Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be NXmx-compliant. On top of the NXmx application definition, Jungfraujoch records a substantial amount of derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and reusable.

This page documents the file layout and the data fields. The operational behaviour of the writer (running, republishing, file finalisation, CBF/TIFF output) is described in jfjoch_writer. The wire format that feeds the writer is described in CBOR messages; fields below frequently correspond one-to-one to CBOR message fields, and that document is a useful companion for their meaning.

1. Motivation: derived metadata and FAIR data

The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both rotation macromolecular crystallography (single- and multi-crystal, including fine-sliced and helical scans) and serial crystallography (stills, grid scans); the same concerns apply to both:

  • Findability. Raw diffraction images carry almost no descriptive metadata about content. Quantities such as background level, number of diffraction spots, or indexing outcome let a user judge the quality and relevance of a dataset before inspecting the raw images.

  • Accessibility at scale. A single experiment can span tens to hundreds of terabytes. Standard retrieval (e.g. HTTP) makes a dataset available but not inspectable — users would otherwise have to download a large fraction of the data just to decide whether it is useful. Compact derived representations make discovery, assessment and reuse feasible.

Because Jungfraujoch couples acquisition with real-time analysis used to steer experiments, transparency and reproducibility of that analysis matter. As a minimum the writer therefore preserves spot-finding and indexing results together with the filters that were applied, and it can retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse.

Two complementary layouts: per-image spots vs. a reflection table

Jungfraujoch stores analysis products in two shapes, matching how each is accessed.

Per-image spot finding / indexing. Spot finding and indexing are inherently image-centric — the natural query is “give me the spots for image n” — and this holds for serial stills and for rotation frames alike. For these products Jungfraujoch adopts a layout similar to the Coherent X-ray Imaging (CXI) data bank (Maia, 2012) and the convention understood by CrystFEL: spot properties (position, intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, with each image allocated room for up to a predefined maximum number of spots. These dense arrays are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved without traversing variable-length structures. The cost is some storage overhead for unused slots (padded with sentinels), which is acceptable for the access pattern.

Integrated reflections. Integrated intensities are naturally a dataset-wide table, which is exactly the model of the NeXus NXreflections base class. This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration results (see §4.2 below). We deliberately do not force spot finding/indexing into a single experiment-wide table: across the hundreds of thousands of patterns typical of serial — or fine-sliced rotation — experiments, that would require aggregating the whole experiment before the spots of one image can be read. We encourage the community to develop standardised NeXus application definitions for image-centric crystallography products that combine NeXus interoperability with the access patterns and scale of modern high-throughput experiments.

2. File layout

A run is written as one master file plus, depending on the format, one or more data files:

<prefix>_master.h5             # NXmx master file (metadata + links / virtual datasets)
+<prefix>_data_000001.h5        # data file: images + per-image analysis
+<prefix>_data_000002.h5
+...
+

The master file is produced by writer/HDF5NXmx.cpp; data files by writer/HDF5DataFile.cpp and its plugins (writer/HDF5DataFilePlugin*.cpp). Files are written to a temporary *.<random>.tmp name and renamed on successful close.

Three master-file variants exist (set via file_format):

Format

Value

Master ↔ data linking

NXmxLegacy (default)

1

One external link in /entry/data per data file (data_000001, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0.

NXmxVDS

2

A single virtual dataset /entry/data/data spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+.

NXmxIntegrated

3

No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format.

In legacy/VDS mode, image-indexed analysis arrays live in the data files and are exposed in the master file through external links or virtual datasets; in integrated mode they are written directly into the single file. Throughout this document a “✓ in master” column marks entries that are visible (directly or via link/VDS) from the master file.

Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or bitshuffle + Zstd; signed integer image datasets use INTx_MIN as the HDF5 fill value (the “masked / no-data” sentinel), unsigned use UINTx_MAX.

Reprocessing output: <prefix>_process.h5

The offline reprocessing tool rugnux (tools/rugnux_cli.cpp) re-runs the full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing dataset and writes its results to a master file named <prefix>_process.h5. This file uses the integrated format, but instead of copying the images its /entry/data/data is a virtual dataset that links back to the original image files (hdf5_source_data → NXmx::LinkToData_ProcessingVDS). The result is a compact, self-describing companion file that holds all the derived analysis (everything in §4) plus a virtual view of the raw images — without duplicating terabytes of data.

This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it processed, while the /entry/data/data VDS still resolves to the original images when they are available. rugnux can also process an equally-spaced subset of images (start/end/stride), producing a down-sampled reference set.

3. NXmx-standard content

The entries below are part of, or valid base classes for, the NXmx application definition. “NXmx” = listed in the application definition; “base” = a valid field of the relevant NeXus base class (NXdetector, NXsample, NXsource) but not in the NXmx required/recommended subset.

/entry (NXentry)

Field

Std

Notes

definition

NXmx

value "NXmx"

start_time

NXmx

arming time

end_time, end_time_estimated

NXmx

approximate end time

File-level HDF5 attributes file_name, file_time, HDF5_Version are also set.

/entry/source (NXsource), /entry/instrument (NXinstrument)

Field

Std

Units

source/name, source/type

NXmx / base

source/current

base

A

instrument/name

NXmx

/entry/instrument/beam (NXbeam)

Field

Std

Units

incident_wavelength

NXmx

angstrom

incident_wavelength_spread

NXmx

angstrom (only if polychromatic)

total_flux

NXmx

Hz

/entry/instrument/attenuator (NXattenuator)

Field

Std

attenuator_transmission

NXmx

/entry/instrument/detector (NXdetector)

Field

Std

Units

depends_on

NXmx

→ transformations/rot3

beam_center_x, beam_center_y

NXmx

pixel

distance

NXmx

m

count_time, frame_time

NXmx

s

sensor_thickness

NXmx

m

sensor_material

NXmx

description

NXmx

threshold_energy

NXmx

eV (EIGER; written only for a single channel)

x_pixel_size, y_pixel_size

base

m

serial_number

base

bit_depth_readout

NXmx

saturation_value

NXmx

flatfield_applied

NXmx

pixel_mask, pixel_mask_applied

NXmx

pixel_mask is [y, x], hard-linked from detectorSpecific/pixel_mask

countrate_correction_applied

NXmx

number_of_cycles

base

frame-summation factor

/entry/instrument/detector/transformations (NXtransformations)

The NXtransformations mechanism (the depends_on chain, transformation_type, vector, offset attributes) is standard. The axis names follow the PyFAI PONI convention chosen by Jungfraujoch (see DETECTOR_GEOMETRY):

Axis

Type

Units

Depends on

translation

translation

m

.

rot1

rotation

rad

translation

rot2

rotation

rad

rot1

rot3

rotation

rad

rot2

The beam centre is encoded in translation (its offset from the sample), not only in the informational beam_center_x/beam_center_y fields. In a _process.h5 written by rugnux these axes carry the refined detector geometry — the refined beam centre folds into translation and the refined tilt into rot1/rot2/rot3; the broker writes the user-provided geometry unchanged.

/entry/instrument/detector/module (NXdetector_module)

data_origin, data_size, fast_pixel_direction, slow_pixel_direction, module_offset — all NXmx (fast/slow_pixel_direction and module_offset carry transformation attributes).

/entry/sample (NXsample)

Field

Std

Units / notes

name

NXmx

depends_on

NXmx

points at the last goniometer / grid-scan axis, or . for stills

temperature

NXmx

K

transformations/ (NXtransformations)

NXmx

rotation axis (e.g. omega) or grid-scan translation; hard-linked as /entry/sample/goniometer

unit_cell

base

[a, b, c, α, β, γ]

ub_matrix

base

[1, 3, 3], Angstrom⁻¹

For a rotation scan the goniometer axis is written as a per-image angle array <axis> plus <axis>_end, scalar <axis>_range_average, <axis>_range_total, and for helical scans <axis>_helical_x/_y/_z. These extra goniometer datasets beyond the bare axis array are Jungfraujoch conveniences.

/entry/data (NXdata)

data (3-D image stack, [n_images, y, x]) with image_nr_low / image_nr_high attributes. In legacy mode this group instead contains one external link data_000001, … per data file.

4. Extensions beyond NXmx

Everything in this section is outside the NXmx standard. Each group is declared with NX_class = NXcollection (the NeXus-sanctioned container for non-standardised content) unless noted. The per-image arrays are indexed by image number, padded to the run length and filled with a sentinel (NaN for floats, -1/0 for integer indices) where a quantity is absent.

4.1 /entry/MX — spot finding and indexing (CXI-style)

The flagship extension. Spot (“peak”) properties are stored as fixed-size [n_images, max_spots] arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as [n_images] vectors. In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master.

Per-spot arrays [n_images, max_spots]:

Dataset

Units

Meaning

Indexing only

peakXPosRaw, peakYPosRaw

pixel

spot position (raw detector frame)

peakTotalIntensity

photons

spot intensity

peakIceRingRes

spot lies in an ice-ring resolution band

peakH, peakK, peakL

Miller indices of the (indexed) spot

✓

peakDistEwaldSphere

Å⁻¹

distance of the spot from the Ewald sphere

✓

peakIndexed

spot fits the indexing solution

✓

peakLattice

lattice the spot belongs to (-1 = unindexed)

✓

Per-image vectors [n_images]:

Dataset

Units

Meaning

nPeaks

number of spots stored for the image (CXI)

strongPixels

strong-pixel count (first spot-finding stage)

peakCountUnfiltered

spots found before filtering

peakCountLowRes

low-resolution spots

peakCountIceRingRes

spots inside ice-ring bands

peakCountIndexed

spots fitting the indexing solution

imageIndexed

image was indexed (0/1)

indexingLatticeCount

number of lattices found for the image

niggliClass

Niggli class of the indexed Bravais lattice (see International Tables for Crystallography A (2016), Vol. A, Table 3.1.3.1)

bravaisLattice

Bravais lattice short code, e.g. aP, mC, oF, tI, hP, hR, cF

profileRadius

Å⁻¹

crystal profile radius

mosaicity

deg

mosaicity estimate

bFactor

Ų

per-image B-factor estimate

resolutionEstimate

Å

diffraction resolution estimate

integratedReflections

number of integrated reflections

bkgEstimate

photons

mean background in the 3–5 Å resolution band

iceRingScore

ratio

strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

beam_corr_x, beam_corr_y

pixel

beam-center correction applied during processing

imageScaleFactor

on-the-fly per-image scale factor g

imageScaleCC

on-the-fly scaling correlation coefficient

imageScaleMosaicity

deg

scaling-model mosaicity

imageScaleBFactor

Ų

scaling-model B-factor

Per-image lattices: latticeIndexed [n_images, 9] (Å) — the real-space lattice (flattened 3×3); latticeIndexedExtra [n_images, max_extra_lattices, 9] (Å) — additional orientation variants.

Run-level summaries (written into the master /entry/MX at finalisation):

Dataset

Units

Meaning

indexing_algorithm

FFBIDX / FFT (CUDA) / FFT (FFTW)

geom_refinement_algorithm

e.g. beam_center

rotationLatticeIndexed

Å

whole-run rotation-indexing lattice ([9])

rotationLatticeIndexedExtra

Å

additional whole-run lattices ([m, 9])

rotationLatticeNiggliClass

Niggli class of the run lattice

imageIndexedMean

mean indexing rate over the run

bkgEstimateMean

photons

mean background over the run

indexedLatticeCount

per-image lattice count summary (master). Note: data files use indexingLatticeCount; readers accept either.

CrystFEL can read the spots directly with:

peak_list = /entry/MX
+peak_list_type = cxi
+

4.2 /entry/reflections — integrated reflections (NXreflections)

Integrated reflections are stored per image as /entry/reflections/image_NNNNNN groups, each declared NX_class = NXreflections. The columns map mostly onto the standard NXreflections base class:

Dataset

Units

NXreflections

Meaning

h, k, l

standard

Miller indices

d

Å

standard

resolution

int_sum

photons

standard

integrated intensity (summation)

int_err

photons

non-standard name

σ of the intensity (standard equivalent: int_sum_errors)

background_mean

photons

standard

mean background under the peak

predicted_x, predicted_y

pixel

name standard, units differ

predicted position. NXreflections predicted_x/_y are physical lengths; the pixel datasets are predicted_px_x/_y

observed_x, observed_y

pixel

name standard, units differ

observed centroid (pixels; standard pixel form is observed_px_x/_y)

observed_frame

standard

image number of the reflection

lp

standard

Lorentz–polarization factor (stored as 1/rlp)

partiality

standard

recorded fraction of the reflection

delta_phi

deg

extension

XDS Δφ: offset from the centre of the current frame

zeta

extension

Lorentz ζ factor (reciprocal-space geometry term)

image_scale_corr

extension

per-image scale correction; I_true = image_scale_corr · int_sum

In the master file these per-image groups are exposed through /entry/reflections external links (VDS/integrated formats).

4.3 /entry/azint — azimuthal integration

Dataset

Shape

Units

Meaning

bin_to_q

[φ_bins, q_bins]

Å⁻¹

q value of each bin

bin_to_two_theta

[φ_bins, q_bins]

deg

2θ of each bin

bin_to_phi

[φ_bins, q_bins]

deg

azimuthal angle of each bin

image

[n_images, φ_bins, q_bins]

per-image integrated profile (NaN for empty bins)

image_std

[n_images, φ_bins, q_bins]

per-bin standard deviation

image_count

[n_images, φ_bins, q_bins]

pixels contributing per bin

map

[y, x]

pixel→bin mapping (master file only)

4.4 /entry/roi — regions of interest (per-image results)

/entry/roi/<roi_name> has one sub-group per configured ROI, holding the per-image result vectors [n_images]. These are written into the data files; in VDS mode they are exposed from the master file through virtual datasets, and in integrated mode they are in the single file. (In legacy mode they remain only in the data files.)

Dataset

Meaning

max

maximum pixel value in the ROI

sum

sum of pixel values

sum_sq

sum of squared pixel values

npixel

number of valid pixels

x, y

intensity-weighted centroid

4.4.1 /entry/roi_defs — ROI definitions (master file)

The dataset-wide ROI definitions (geometry, fixed for the whole acquisition) live in the master file under a separate /entry/roi_defs group — kept apart from /entry/roi above so that older readers, which iterate /entry/roi, are unaffected by these entries. One sub-group /entry/roi_defs/<roi_name> per ROI:

Dataset

Meaning

bit_index

which bit of roi_map (below) marks this ROI

type

box, circle or azim

min_x_pxl, max_x_pxl, min_y_pxl, max_y_pxl

box bounds (type box)

center_x_pxl, center_y_pxl, radius_pxl

circle (type circle)

q_min_recipA, q_max_recipA

Q range (type azim)

phi_min_deg, phi_max_deg

azimuthal-angle sector (type azim, omitted for a full ring)

/entry/roi_defs/roi_map [y, x] is a uint16 per-pixel bitmask: bit bit_index is set for every pixel belonging to that ROI, so an ROI’s footprint can be recovered exactly.

4.5 /entry/image — per-image pixel statistics

[n_images] vectors: max_value, min_value (viable min/max, excluding error/saturated pixels), error_pixels, saturated_pixels, pixel_sum. Surfaced in the master file under /entry/image.

4.6 /entry/profiling — per-image timing

[n_images] vectors in seconds: spotFindingTime, indexingTime, integrationTime, refinementTime, processingTime, braggPredictionTime, preprocessingTime, compressionTime, azIntTime, indexAnalysisTime, imageScaleTime.

4.7 /entry/detector — acquisition diagnostics (data file)

A convenience NXcollection in the data file (note: distinct from the standard /entry/instrument/detector). In integrated format these datasets are written under /entry/instrument/detector/detectorSpecific instead.

Dataset

Meaning

timestamp, exptime

per-image timestamp and exposure time

number

image number (original number if image rejection was used)

det_info

JUNGFRAU debug field

storage_cell_image

storage-cell number

rcv_delay, rcv_free_send_buffers

receiver internal diagnostics

packets_expected, packets_received

UDP packets per image

data_collection_efficiency_image

received / expected packet ratio

4.8 /entry/xfel — pulsed-source metadata

[n_images] vectors pulseID and eventCode, written for pulsed sources (e.g. SwissFEL).

4.9 Other collections

Path

Class

Content

/entry/instrument/detector/detectorSpecific

NXcollection

Dectris-style detector metadata + Jungfraujoch fields: x_pixels_in_detector, y_pixels_in_detector, nimages, ntrigger, nimages_collected, nimages_written, data_collection_efficiency, max_receiver_delay, storage_cell_number, storage_cell_delay [ns], software_git_commit, software_git_date, jfjoch_release, jfjoch_writer_release, summation_mode, detect_ice_rings, gain_file_names, data_reduction_factor_serialmx, adu_histogram/, data_collection_efficiency_image

/entry/instrument/detector/calibration

NXcollection

per-channel pedestal / calibration images (bitshuffle-compressed)

/entry/instrument/fluorescence

NXcollection

XRF spectrum: energy [eV], data

/entry/user

NXcollection

scalar values supplied under header_appendix.hdf5

4.10 Non-standard fields inside the NXmx detector group

A few extension scalars are written inside the otherwise-standard /entry/instrument/detector group for compatibility with existing tooling:

Field

Units

Meaning

detector_distance

m

duplicate of distance (Dectris/Neggia compatibility)

detector_number

detector identifier (Dectris convention)

error_value

masked/error pixel sentinel (NXmx standard would be underload_value)

bit_depth_image

stored image bit depth (NXmx standard is bit_depth_readout)

acquisition_type

always triggered (Dectris convention)

jungfrau_conversion_applied

JUNGFRAU photon/keV conversion applied

jungfrau_conversion_factor

eV

conversion factor

geometry_transformation_applied

module→full-detector geometry applied

4.11 User-supplied metadata: header_appendix and image_appendix

Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two free-form JSON fields in the /start request (broker/jfjoch_api.yaml) provide this without any schema change; both accept any valid JSON:

Field

Carried in

Persisted to HDF5?

header_appendix

the start message, under user_data.user (see CBOR)

no — except the hdf5 sub-object (below)

image_appendix

every image message, as user_data

no

Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, republished analysis, viewers), so they are the recommended channel for facility- or beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has no dedicated API field.

Persisting selected values to HDF5. header_appendix is normally not written to the master file. As an exception, if it contains a key hdf5 whose value is a JSON object of scalars (strings and numbers — no arrays or nested objects), the writer stores each entry under /entry/user/<key>.

For example, a /start request containing:

{
+  "header_appendix": {
+    "proposal": "p20001",
+    "operator": "jdoe",
+    "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 }
+  },
+  "image_appendix": { "trigger_source": "external" }
+}
+

forwards the whole header_appendix as user_data.user on the start message and {"trigger_source": "external"} as user_data on every image message, and writes three scalars into the master file:

/entry/user/beamline          = "X06SA"
+/entry/user/ring_mode         = "top-up"
+/entry/user/attenuator_foils  = 2
+

5. Notes

  • Units are written as the HDF5 units attribute on the dataset (e.g. m, eV, deg, Angstrom, Angstrom^-1, Angstrom^2, pixel, s).

  • Sentinels. Missing per-image values are NaN (floats) or -1/0 (integer indices); image pixels use INTx_MIN / UINTx_MAX.

  • Master vs data file. In legacy/VDS formats the analysis arrays physically live in the data files; the master file links to them (external links in legacy, virtual datasets in VDS). In the integrated format there are no data files and everything is in one place.

  • CXI / CrystFEL. /entry/MX follows the CXI peak-list convention; see CXI file format.

\ No newline at end of file diff --git a/IMAGE_STREAM.html b/IMAGE_STREAM.html new file mode 100644 index 00000000..d12dbdeb --- /dev/null +++ b/IMAGE_STREAM.html @@ -0,0 +1,30 @@ + Data streams — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Data streams

Jungfraujoch process (jfjoch_broker) operates three outputs. All three can be operated/enabled independently. These are:

  • Image - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket)

  • Preview - images with metadata at a reduced frame rate (PUB socket)

  • Metadata - only metadata for all the images, bundled into packages (PUB socket)

Image stream

Images (with metadata) are serialized as CBOR image message. The stream will also include CBOR start message, calibration messages and end message with run metadata.

If file_prefix is not provided for a data collection, images won’t be sent to image stream (or its HDF5/CBOR replacements).

Splitting image stream

Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. All sockets will forward start and end messages. Only first socket will forward calibration messages and will be marked to write master file.

ZeroMQ image stream

This is using PUSH ZeroMQ socket(s). It should be strictly avoided to have multiple receivers connected to one PUSH ZeroMQ socket. ZeroMQ will send the images in a round-robin basis to the receivers. In this case start and end messages will end up only with one receiver. Instead, Jungfraujoch feature of multiple sockets should be used. For ZeroMQ image stream, each writer connects to a different port.

Behavior is as following:

  • Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer.

  • Calibration message is sent to the first socket only, with timeout of 1s.

  • Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped).

  • End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded.

The format is generally interchangeable with DECTRIS Stream2 format.

ZeroMQ configuration

ZeroMQ image stream is configured in the broker JSON configuration file under the zeromq_settings section:

{
+  "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"],
+  "send_watermark": 100,
+  "send_buffer_size": 67108864,
+  "writer_notification_socket": "tcp://192.168.0.1:*"
+}
+

  • image_socket: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (tcp://, ipc://). 0.0.0.0 binds on all network interfaces.

  • send_watermark (optional): ZeroMQ send high-water mark (number of outstanding messages per socket).

  • send_buffer_size (optional): OS-level send buffer size for the ZeroMQ socket.

  • writer_notification_socket (optional): see Writer notification socket below.

TCP/IP image stream

This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication between the data collection and the writer, and is therefore more robust than ZeroMQ.

For TCP/IP image stream, Jungfraujoch listens on a single TCP port and all writers connect to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic KEEPALIVE frames when no data collection is active to detect dead connections; writers are expected to respond with a KEEPALIVE pong.

Using * as port number (e.g. tcp://127.0.0.1:*) is supported — the OS assigns a free port and the actual bound address can be queried via GetAddress().

Payloads for START, DATA, CALIBRATION and END frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages.
ACK, CANCEL, and KEEPALIVE are control frames (no CBOR payload).

The data collection lifecycle on each connection follows: START → CALIBRATION (socket 0 only) → DATA (repeated) → END

If a START ACK fails on any connection, Jungfraujoch sends CANCEL to all already-started connections and rolls back.

For each frame:

  1. Read one TcpFrameHeader (fixed size, 64-byte aligned).

  2. Validate magic (0x4A464A54 / "JFJT") and version (2).

  3. Read payload_size bytes (if non-zero).

When image stream is split into multiple connections:

  • START and END are sent on all connections,

  • CALIBRATION is sent only on connection 0,

  • DATA frames are distributed by file grouping: connection index = (image_number / images_per_file) % num_connections.

TCP/IP configuration

TCP/IP image stream is configured in the broker JSON configuration file under the tcp_settings section:

{
+  "image_socket": "tcp://192.168.0.1:9100",
+  "nwriters": 2,
+  "send_buffer_size": 67108864
+}
+

  • addr: listen address in tcp://<IP>:<port> format. 0.0.0.0 binds on all interfaces. * as port selects a random free port.

  • nwriters (optional): maximum number of simultaneous writer connections accepted.

  • send_buffer_size (optional): OS-level SO_SNDBUF size for accepted connections.

ACK handling

ACK handling is mandatory for correct operation:

  • START must be acknowledged (ACK with ack_for=START) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered.

  • END must be acknowledged (ack_for=END) on each connection within 10 seconds for successful completion.

  • CANCEL should be acknowledged during rollback paths (500ms timeout).

  • DATA should be acknowledged for every frame. A DATA ACK with FATAL flag set reports a downstream error (e.g. disk full) which is propagated to jfjoch_broker via Finalize(). A failed DATA ACK does not break the TCP connection on its own — data continues to flow.

  • CALIBRATION is not acknowledged at this time.

  • KEEPALIVE frames are not acknowledged via ACK; the writer responds with a KEEPALIVE pong frame instead.

Keepalive

When no data collection is active, Jungfraujoch sends KEEPALIVE frames approximately every 5 seconds on each persistent connection. Writers should respond with a KEEPALIVE frame (pong). OS-level TCP keepalive is also enabled (TCP_KEEPIDLE=30s, TCP_KEEPINTVL=10s, TCP_KEEPCNT=3) as a secondary safety net. Dead connections are automatically removed from the pool.

Zero-copy transmission

On Linux, large payload transmission (DATA and CALIBRATION frames) can use kernel TCP zero-copy (SO_ZEROCOPY/MSG_ZEROCOPY) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal send() behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread.

Frame types

Value

Name

Purpose

1

START

Start-of-run metadata

2

DATA

One image payload

3

CALIBRATION

Calibration payload

4

END

End-of-run metadata

5

ACK

Acknowledgement / error reporting

6

CANCEL

Cancel run initialization/stream

7

KEEPALIVE

Connection liveness probe/pong

TCP frame header (TcpFrameHeader)

Field

Type

Description

magic

uint32_t

Protocol magic (0x4A464A54, "JFJT")

version

uint16_t

Protocol version (2)

type

uint16_t

Frame type (see table above)

image_number

uint64_t

Image index for DATA frames

payload_size

uint64_t

Number of payload bytes after header

socket_number

uint32_t

Connection index in split-stream mode

flags

uint32_t

ACK flags (OK, FATAL, HAS_ERROR_TEXT)

run_number

uint64_t

Run identifier

ack_processed_images

uint32_t

In ACK: number of images processed by receiver

ack_code

uint16_t

In ACK: error/status code

ack_for

uint16_t

In ACK: frame type being acknowledged

ack_fifo_occupancy

uint16_t

In ACK: occupancy of input FIFO in the jfjoch_writer

ack_fifo_max_occupancy

uint64_t

In ACK: max occupancy of input FIFO

The header is 64-byte aligned (alignas(64)).

ACK semantics

  • ACK frames use ack_for to indicate which frame type is acknowledged.

  • flags:

    • OK (bit 0): operation accepted/successful,

    • FATAL (bit 1): receiver reports unrecoverable error (primarily for DATA),

    • HAS_ERROR_TEXT (bit 2): ACK payload contains UTF-8 error text.

  • ack_code can be used to categorize errors:

Code

Name

Meaning

0

None

No error

1

StartFailed

START processing failed

2

DataWriteFailed

Image write failed

3

EndFailed

END processing failed

4

DiskQuotaExceeded

Disk quota exceeded

5

NoSpaceLeft

No space left on device

6

PermissionDenied

Permission denied

7

IoError

General I/O error

8

ProtocolError

Protocol-level error

Image stream replacement

Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting “None” image pusher for all the measurements.

Writer notification socket

The writer notification socket is used only with ZeroMQ image stream. Since ZeroMQ is asynchronous, jfjoch_broker does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back.

For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame.

To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with writer_notification_socket entry:

{
+  "writer_notification_socket":"tcp://192.168.0.1:*"
+}
+

Such entry will create PULL socket on 192.168.0.1 network interface listening on one, random TCP port. When data processing is started, the image stream will send CBOR start message. This message will include information on writer_notification_zmq_addr, which needs to be used by downstream code. Since the start message must reference the address of jfjoch_broker host, notification socket should always listen on a particular network interface, and should not be configured with placeholder address 0.0.0.0. It is, however, OK to use placeholder :* for network port, as it will be substituted for the one chosen by ZeroMQ.

For every image stream socket, downstream code must send the following message to the PULL socket:

{
+  "run_number":135,
+  "run_name": "lysozyme_1",
+  "socket_number": 1,
+  "processed_images":250,
+  "ok": true
+}
+

Here run_number, run_name and socket_number must match information from the start message. ok is boolean confirming if the writing process was OK. processed_images is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. If not, it is possible to include error message:

{
+  "run_number":135,
+  "run_name": "lysozyme_1",
+  "socket_number": 1,
+  "processed_images": 0,
+  "ok": false,
+  "error": "Permission error"
+}
+

This way errors from the downstream code are propagated to jfjoch_broker.

If writer notification socket is configured, but downstream code doesn’t send proper notification, jfjoch_broker will time out after 60 seconds producing an error message.

Preview stream

Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. Images are serialized as CBOR image message. The stream will also include CBOR start message and end message with run metadata. Only start and image messages are sent.

This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope with the messages, it will always receive the last generated message (no backlog). For this reason it is also recommended to use the same option on receiver side.

Given PUB socket properties, it is possible to connect multiple viewers to a single socket — all the viewers should receive all the images sent.

Metadata stream

Jungfraujoch can also send pure metadata for the purpose of archiving such information. Metadata are serialized as CBOR metadata message. This is very similar as image message, but excludes the actual image array and spot positions. As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. Order of images within bundle, as well a size of the bundle, are not guaranteed. The stream will also include CBOR start message and end message with run metadata.

This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected.

\ No newline at end of file diff --git a/JFJOCH_BROKER.html b/JFJOCH_BROKER.html new file mode 100644 index 00000000..12cc08e0 --- /dev/null +++ b/JFJOCH_BROKER.html @@ -0,0 +1,126 @@ + jfjoch_broker — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

jfjoch_broker

jfjoch_broker is the main service for the Jungfraujoch application. It is responsible for:

  • Providing user interface via HTTP and OpenAPI

  • Configuring FPGA firmware

  • Building images from FPGA output and forwarding the results over ZeroMQ

External interfaces

Broker operates four external interfaces.

Image stream ZeroMQ PULL socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream processing. See details here.

Preview stream ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details here.

Metadata stream ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details here.

Configuration, status and results interface HTTP/REST interface described in the OpenAPI format. Description of the API is presented in the OpenAPI description.

Broker configuration

jfjoch_broker requires JSON configuration files. The file is described by OpenAPI structure jfjoch_settings defined in jfjoch_api.yaml file. It is recommended to go through example files in the etc/.

Example with all fields:

{
+  "pcie": [
+    {
+      "blk": "/dev/jfjoch0",
+      "ipv4": "10.1.1.7"
+    },
+    {
+      "blk": "/dev/jfjoch1",
+      "ipv4": "10.1.1.8"
+    }
+  ],
+  "zeromq": {
+    "send_watermark": 100,
+    "send_buffer_size": 1024,
+    "image_socket": [
+      "tcp://1.2.3.4:5000",
+      "tcp://1.2.3.4:5001"
+    ],
+    "writer_notification_socket": "tcp://1.3.4.6:7000"
+  },
+  "instrument": {
+    "source_name": "Swiss Light Source",
+    "source_type": "Synchrotron X-ray Source",
+    "instrument_name": "X06SA",
+    "pulsed_source": false,
+    "electron_source": false
+  },
+  "detector": [
+    {
+      "description": "EIGER 1M",
+      "serial_number": "E1M-01",
+      "type": "EIGER",
+      "high_voltage_V": 150,
+      "udp_interface_count": 1,
+      "module_sync": true,
+      "sensor_thickness_um": 320,
+      "calibration_file": [
+        "gainMaps.bin"
+      ],
+      "hostname": [
+        "e1m-01",
+        "e1m-02"
+      ],
+      "readout_time_us": 3,
+      "sensor_material": "Si",
+      "tx_delay": [
+        0,1
+      ],
+      "base_data_ipv4_address": "10.10.10.50",
+      "standard_geometry": {
+        "nmodules": 1,
+        "gap_x": 8,
+        "gap_y": 36,
+        "modules_in_row": 1
+      },
+      "custom_geometry": [
+        {
+          "x0": 0,
+          "y0": 0,
+          "fast_axis": "Xp",
+          "slow_axis": "Xp"
+        }
+      ],
+      "mirror_y": true
+    }
+  ],
+  "detector_settings": {
+    "frame_time_us": 450,
+    "count_time_us": 0,
+    "internal_frame_generator": false,
+    "internal_frame_generator_images": 1,
+    "detector_trigger_delay_ns": 0,
+    "timing": "auto",
+    "eiger_threshold_keV": 6.0,
+    "jungfrau_pedestal_g0_frames": 2000,
+    "jungfrau_pedestal_g1_frames": 300,
+    "jungfrau_pedestal_g2_frames": 300,
+    "jungfrau_pedestal_g0_rms_limit": 100,
+    "jungfrau_pedestal_min_image_count": 128,
+    "jungfrau_storage_cell_count": 1,
+    "jungfrau_storage_cell_delay_ns": 5000,
+    "jungfrau_fixed_gain_g1": false,
+    "jungfrau_use_gain_hg0": false
+  },
+  "azim_int": {
+    "polarization_factor": -1,
+    "solid_angle_corr": true,
+    "high_q_recipA": 0,
+    "low_q_recipA": 0,
+    "q_spacing": 0
+  },
+  "image_format": {
+    "summation": true,
+    "geometry_transform": true,
+    "jungfrau_conversion": true,
+    "jungfrau_conversion_factor_keV": 0.001,
+    "bit_depth_image": 16,
+    "signed_output": true,
+    "mask_module_edges": true,
+    "mask_chip_edges": true
+  },
+  "image_buffer_MiB": 2048,
+  "receiver_threads": 64,
+  "frontend_directory": "/usr/share/jfjoch/frontend",
+  "image_pusher": "ZeroMQ",
+  "zeromq_metadata": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4357"
+  },
+  "zeromq_preview": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4356"
+  }
+}
+

Setting up a local test for Jungfraujoch

For development, it is possible to set up a local installation of Jungfraujoch. This will work without FPGA installed in the computer and allows to test Jungfraujoch software layer, including ZeroMQ streaming and file writing.

The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore very low, as fixed-point calculations have large performance penalty on CPU. In the CPU simulation mode, one can simulate using only a single FPGA device.

To run the test:

Compile Jungfraujoch with frontend

mkdir build
+cd build
+cmake ..
+make jfjoch
+make frontend
+

Alternatively, for RHEL8 system, you can use RPM generated by automated pipeline. Solely jfjoch one is enough. In this case - it is necessary to update etc/broker_local.json file with frontend path in /usr/share/jfjoch/frontend.

Start service

Start broker:

cd build/broker
+./jfjoch_broker ../../etc/broker_local.json 5232
+

Run tests

To run test a Python script is provided:

cd tests/test_data
+python jfjoch_broker_test.py
+

The script will initialize Jungfraujoch, import test image and start data collection.

Expected result

You can observe online data analysis by opening the following web page: http://localhost:5232. Also, a dataset with images should be written in the build/broker directory.

\ No newline at end of file diff --git a/JFJOCH_VIEWER.html b/JFJOCH_VIEWER.html new file mode 100644 index 00000000..7fcb198c --- /dev/null +++ b/JFJOCH_VIEWER.html @@ -0,0 +1,17 @@ + jfjoch_viewer — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

jfjoch_viewer

jfjoch_viewer is the interactive desktop application of Jungfraujoch. It opens diffraction datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal integration, per-image statistics), and can follow a live data collection by syncing with a running jfjoch_broker over its HTTP interface.

It is a standalone Qt 6 application, distributed pre-built on the Gitea release page and in the Jungfraujoch RPM/APT repositories (see Deployment).

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

Displayed on screen (results not saved to disk)

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Functionality

  • Opens HDF5 files written by jfjoch_writer (*_master.h5) and the *_process.h5 files produced by rugnux. It also opens NXmx files written by DECTRIS detectors, though that path has had only limited testing.

  • Runs an embedded data-processing pipeline — the same analysis code as the rest of Jungfraujoch — performing spot finding, indexing and integration on the displayed images. Results are shown on screen but are not saved to disk.

  • Auxiliary windows and panels: image list, image metadata, spot list, reflection list, per-region-of-interest statistics, the azimuthal-integration profile, and dataset-info charts.

  • User-mask editing: build a user mask interactively, clear it, save it as TIFF, or upload it to a connected server.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. On systems with a GPU, use the CUDA build (provided as separate RPM/APT repositories) for the embedded indexing and integration; the non-CUDA build runs the same pipeline on the CPU at much lower throughput.

Opening data

  • File ▸ Open (Ctrl+O) — open a local HDF5 file.

  • File ▸ Open HTTP (Ctrl+H) — connect to a jfjoch_broker HTTP endpoint to follow a live collection. The dialog defaults to host localhost and port 8080; these defaults can be overridden with the environment variables JUNGFRAUJOCH_HTTP_HOST and JUNGFRAUJOCH_HTTP_PORT.

  • Command line — jfjoch_viewer <file.h5> opens a file (or an http://host:port URL) on start-up. --dbus <true|false> (-d) enables or disables the D-Bus interface (default: enabled); --help and --version behave as usual.

D-Bus interface

When enabled, the viewer registers the D-Bus interface ch.psi.jfjoch_viewer, so other processes can drive it:

  • LoadFile(filename, image_number=0, summation=1) — open a file (or an http://host:port URL) and display the given image.

  • LoadImage(image_number, summation=1) — navigate to an image in the already-open dataset.

summation sums that many consecutive images before display.

Building from source on Windows

jfjoch_viewer is the one Jungfraujoch component that is cross-platform: it builds on Windows 11 with MSVC and the full CUDA GPU path. (The rest of Jungfraujoch — broker, receiver, FPGA host — is Linux-only.) There is no pre-built Windows package yet, so build it from source. On Windows the build is automatically restricted to the viewer and the libraries it needs (JFJOCH_VIEWER_ONLY is forced on), and the remaining dependencies are fetched and built automatically (the first configure needs network access).

Verified toolchain:

  • Windows 11

  • Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC

  • CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path

  • Qt 6.11 for MSVC (msvc2022_64), including the Qt Charts module — e.g. C:\Qt\6.11.1\msvc2022_64

  • CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of the box — it comes with the C++ workload, so there is nothing extra to install. Any recent standalone CMake (from cmake.org, or the one bundled with Qt in C:\Qt\Tools\CMake_64) works too.

  • zlib and Eigen — the two libraries not auto-fetched on Windows. Build/install both into one prefix (here C:\deps) and point CMake at it:

    :: static zlib
+git clone --branch v1.3.1 https://github.com/madler/zlib
+cmake -G Ninja -S zlib -B zlib-build -DCMAKE_INSTALL_PREFIX=C:/deps
+cmake --build zlib-build --target install
+:: Eigen 3.4 (header-only) -- install just the headers with `cmake --install`; the BLAS/LAPACK/test
+:: targets are disabled since they are not needed (and fail to build under MSVC). Use the 3.4 series:
+:: the project requests find_package(Eigen3 3.4), which Eigen's same-major rule rejects for 5.x.
+git clone --branch 3.4.0 https://gitlab.com/libeigen/eigen.git
+cmake -G Ninja -S eigen -B eigen-build -DCMAKE_INSTALL_PREFIX=C:/deps ^
+  -DEIGEN_BUILD_BLAS=OFF -DEIGEN_BUILD_LAPACK=OFF -DEIGEN_BUILD_DOC=OFF -DBUILD_TESTING=OFF
+cmake --install eigen-build
+

  • Optional: NSIS to build the .exe installer.

Configure and build from an x64 Native Tools Command Prompt for VS 2026 (so cl, nvcc and ninja are on PATH):

cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^
+  -DCMAKE_PREFIX_PATH="C:/deps;C:/Qt/6.11.1/msvc2022_64"
+cmake --build build-win --target jfjoch_viewer
+

Notes:

  • CMAKE_PREFIX_PATH (the C:/deps prefix plus Qt) is the only required flag — CMake finds zlib and Eigen from the prefix, so no separate -DZLIB_ROOT is needed.

  • The CUDA toolchain is located automatically from the CUDA_PATH environment variable that the CUDA installer sets (or from nvcc on PATH). Pass -DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe" only if nvcc is installed in a nonstandard location and is not found.

  • For a machine without an NVIDIA GPU, add -DJFJOCH_USE_CUDA=OFF: the viewer then runs the same pipeline on the CPU (FFTW indexer) at lower throughput.

To produce a self-contained installer (bundles the Qt runtime via windeployqt, the analysis CLIs, and — on the CUDA build — the cuFFT runtime DLL, so the target host needs no Qt and no CUDA toolkit, only an NVIDIA GPU driver), with NSIS installed:

cd build-win
+cpack
+

The NSIS generator is selected automatically on Windows (no -G needed). The installer filename and the Add/Remove Programs entry mark the CUDA/CPU variant, while the install folder and Start Menu group stay plain Jungfraujoch (the two variants share an install location and replace each other — CUDA is a strict superset):

Build

Installer file

Add/Remove Programs

CUDA (default)

jfjoch-<version>-win64-cuda<major>.exe

Jungfraujoch (CUDA)

-DJFJOCH_USE_CUDA=OFF

jfjoch-<version>-win64-cpu.exe

Jungfraujoch (CPU)

<major> is the CUDA toolkit major version (e.g. cuda13). The cuFFT DLL is ~256 MB, so the CUDA installer is correspondingly larger — hence the variant tag in the filename.

\ No newline at end of file diff --git a/JFJOCH_WRITER.html b/JFJOCH_WRITER.html new file mode 100644 index 00000000..60a41ac7 --- /dev/null +++ b/JFJOCH_WRITER.html @@ -0,0 +1,68 @@ + jfjoch_writer — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

jfjoch_writer

jfjoch_writer is NeXus compliant HDF5 file writer.

Acknowledgements

  • Zdenek Matej (MAX IV)

  • Felix Engelmann (MAX IV) for testing and multiple improvement suggestions.

Running directory

Writer needs to be running in base directory for writing files - file_prefix will be always relative in regard to writer running directory. Writer detects and protects for basic security issues, like file_prefix starting with a slash, or starting with ../, or containing /../.

Usage

Writer needs to be started as a background service, with the following command:

jfjoch_writer {options} <address to connect via ZeroMQ to DCU>
+
+Options:
+-R<int> | --root_dir=<int>           Root directory for file writing
+-H<int> | --http_port=<int>          HTTP port for statistics
+-r<int> | --zmq_repub_port=<int>     ZeroMQ port for PUSH socket to republish images
+-f<int> | --zmq_file_port=<int>      ZeroMQ port for PUB socket for notifications on finalized files
+-w<int> | --rcv_watermark=<int>      Receiving ZeroMQ socket watermark (default = 100)
+-W<int> | --repub_watermark=<int>    Republish ZeroMQ socket watermark (default = 1000)
+

for example:

jfjoch_writer -H5234 tcp://dcu-address:5400 
+

HTTP interface

Writer has dedicated status interface via HTTP. It allows for two operations:

  • check state of the writer to check if the writer is properly synchronized with DCU (e.g., that file_prefix agrees with what was set on the DCU) and monitor progress.

  • cancel writing this will close all the HDF5 files being written and restart writer - the option should be used only if DCU process was terminated or disconnected, it SHOULD NOT be used as standard cancellation procedure (when DCU received cancel command it should properly finish writing as well)

Republish

Republish creates a PULL socket on the writer, where all the messages are republished for further use by data analysis pipeline. Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won’t be republished. In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won’t be retried.

Republish functionality is optional, if republish port number is omitted this functionality is not enabled.

Overwriting files

When jfjoch_writer creates a HDF5 file, it first adds suffix .<random>.tmp. Random value depends on current time-stamp and likely will be different from each file of the particular series. After file is all saved and closed, it is renamed to remove the suffix. By default, renaming won’t happen if this would overwrite existing file. However, this behavior can be changed by setting overwrite parameter to true in the file writer configuration.

When the overwrite conflict is reported

An existing output file is a fatal condition (unless overwrite is true). When it is detected depends on whether the transport between the broker and the writer has a back-channel to report the failure before acquisition starts:

  • Direct HDF5 pusher and TCP writer (back-channel available). The conflict is detected at start: the writer that owns the master file checks whether it already exists and refuses to start. The direct pusher raises the error in-process; the TCP writer returns a START-failure acknowledgement. Either way the broker learns immediately and aborts the data collection before the detector is armed — no images are taken and nothing is written. Only the master file is checked up front: in a multi-writer setup the per-image data files are staggered across writers, and checking them at start would make each writer inspect files it never writes (and race the writers that do). Data-file conflicts are instead caught by their owning writer at the final rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker.

  • ZeroMQ writer (no back-channel). The ZeroMQ image stream is fire-and-forget: the writer has no way to tell the broker to stop, and the broker would keep streaming images regardless. The writer therefore does not fail at start. It writes the whole series to the .<random>.tmp files as usual and only fails at the final rename, leaving the .tmp files on disk. This is deliberate: the acquired images are preserved (in .tmp form) rather than being dropped by a writer that aborted mid-stream. Rename the .tmp files by hand, or re-run with overwrite set, to recover them.

Finalized files information

Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure:

{
+  "filename": <string>: HDF5 data file name (relative to writer root directory),
+  "nimages": <int> number of images in the file (counting from 1!),
+  "file_number": <int> number of file within the acquisition,
+  "sample_name": <string> name of sample,
+  "run_name": <string> name of run,
+  "run_number": <int> number of run,
+  "experiment_group": <string> number of p-group / proposal (optional),
+  "user_data": <any json> user_data,
+  "beam_x_pxl": <float> beam center (X) in pixels,
+  "beam_y_pxl": <float> beam center (Y) in pixels,
+  "detector_distance_m": <float> detector distance (X) in m,
+  "detector_height_pxl": <int> detector size (X) in pixels,
+  "detector_width_pxl": <int> detector size (Y) in pixels,
+  "incident_energy_eV": <float> photon energy of the X-ray beam,
+  "pixel_size_m": <float> pixel size in meter (assuming pixel X == Y),
+  "saturation": <int> this count and higher mean saturation,
+  "space_group_number": <int> space group number (optional),
+  "underload": <int> pixels with this count should be excluded,
+  "unit_cell": <optinal> unit cell dimensions in Angstrom/degree {
+    "a": <float>, "b": <float>, "c": <float>,
+    "alpha": <float>, "beta": <float>, "gamma": <float>
+  },
+}
+

user_data is defined as header_appendix in the /start operation in the jfjoch_broker. Other metadata are also carried over from /start operation.

If the header_appendix is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. For example header_appendix of {"param1": "test1", "param2": ["test1", "test2"]}, than example message will look as follows:

{
+  "filename": "dataset_name_data_000001.h5",
+  "nimages": 1000,
+  "file_number": 0,
+  "sample_name": "lysozyme",
+  "run_name": "lyso_cryo",
+  "run_number": 25,
+  "experiment_group": "p00001",
+  "beam_x_pxl": 1200,
+  "beam_y_pxl": 1500,
+  "detector_distance_m": 0.155,
+  "detector_height_pxl": 2164,
+  "detector_width_pxl": 2068,
+  "image_time_s": 0.001,
+  "nimages": 2,
+  "incident_energy_eV": 12400.0,
+  "pixel_size_m": 7.5e-05,
+  "saturation": 32766,
+  "space_group_number": 96,
+  "underload": -32768,
+  "unit_cell": {
+    "a": 78.0,
+    "alpha": 90.0,
+    "b": 78.0,
+    "beta": 90.0,
+    "c": 39.0,
+    "gamma": 90.0
+  },
+  "user_data": {
+    "param1": "test1", 
+    "param2": ["test1", "test2"]
+  }
+}
+

Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled.

HDF5 file structure

Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics and timing) stored beyond the NXmx standard. The complete file layout — master vs data files, the three format variants (NXmxLegacy, NXmxVDS, NXmxIntegrated), every NXmx field that is populated and every Jungfraujoch extension — is documented in HDF5 / NeXus data format.

If data collection was configured with a header_appendix containing a key hdf5 whose value is a JSON object of numbers and strings, those entries are written to /entry/user.

Other formats (CBF and TIFF)

In addition to HDF5 format, Jungfraujoch allows to save images in the Crystallographic Binary File (CBF) format. CBF files are written according to miniCBF format, with only basic header, and always with 32-bit signed integer format. Dynamic range is reduced to max 2^24, negative numbers are zeroed, and masked, and/or bad pixels are set to -1.

Also writing to TIFF files is possible, though no metadata are saved in this case.

No file option(s)

There are two options to disable writing of files by the writer:

  • Setting file_prefix to empty string - this will disable sending files on ZeroMQ image socket.

  • Setting file format to NoFile - files are streamed over ZeroMQ socket, but jfjoch_writer will not write anything. This can be useful for debugging purposes, or if you only rely on republishing functionality of the jfjoch_writer

\ No newline at end of file diff --git a/LICENSE.html b/LICENSE.html new file mode 100644 index 00000000..182a9d54 --- /dev/null +++ b/LICENSE.html @@ -0,0 +1,20 @@ + License — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

License

Jungfraujoch software is licensed with GPLv3 license. Jungfraujoch FPGA is licensed with CERN OHL-S license (see FPGA license).

GNU GENERAL PUBLIC LICENSE

Version 3, 29 June 2007

Copyright (C) 2007 Free Software Foundation, Inc. https://fsf.org/ Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.

Preamble

The GNU General Public License is a free, copyleft license for software and other kinds of works.

The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program–to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.

When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.

To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.

For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.

Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.

For the developers’ and authors’ protection, the GPL clearly explains that there is no warranty for this free software. For both users’ and authors’ sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.

Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users’ freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.

Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.

The precise terms and conditions for copying, distribution and modification follow.

TERMS AND CONDITIONS

  1. Definitions.

“This License” refers to version 3 of the GNU General Public License.

“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.

“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.

To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.

A “covered work” means either the unmodified Program or a work based on the Program.

To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.

To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.

An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.

  1. Source Code.

The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.

A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.

The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.

The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work’s System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.

The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.

The Corresponding Source for a work in source code form is that same work.

  1. Basic Permissions.

All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.

You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.

Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.

  1. Protecting Users’ Legal Rights From Anti-Circumvention Law.

No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.

When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work’s users, your or third parties’ legal rights to forbid circumvention of technological measures.

  1. Conveying Verbatim Copies.

You may convey verbatim copies of the Program’s source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.

You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.

  1. Conveying Modified Source Versions.

You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:

a) The work must carry prominent notices stating that you modified it, and giving a relevant date.

b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.

c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.

d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.

A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation’s users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.

  1. Conveying Non-Source Forms.

You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:

a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.

b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.

c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.

d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.

e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.

A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.

A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.

“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.

If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).

The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.

Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.

  1. Additional Terms.

“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.

When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.

Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:

a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or

b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or

c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or

d) Limiting the use for publicity purposes of names of licensors or authors of the material; or

e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or

f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.

All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.

If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.

Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.

  1. Termination.

You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).

However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.

Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.

Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.

  1. Acceptance Not Required for Having Copies.

You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.

  1. Automatic Licensing of Downstream Recipients.

Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.

An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party’s predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.

You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.

  1. Patents.

A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor’s “contributor version”.

A contributor’s “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.

Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor’s essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.

In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.

If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient’s use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.

If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.

A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.

Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.

  1. No Surrender of Others’ Freedom.

If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.

  1. Use with the GNU Affero General Public License.

Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.

  1. Revised Versions of this License.

The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.

Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.

If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy’s public statement of acceptance of a version permanently authorizes you to choose that version for the Program.

Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.

  1. Disclaimer of Warranty.

THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.

  1. Limitation of Liability.

IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

  1. Interpretation of Sections 15 and 16.

If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.

END OF TERMS AND CONDITIONS

How to Apply These Terms to Your New Programs

If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.

To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.

<one line to give the program's name and a brief idea of what it does.>
+Copyright (C) <year>  <name of author>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+

Also add information on how to contact you by electronic and paper mail.

If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:

<program>  Copyright (C) <year>  <name of author>
+This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+This is free software, and you are welcome to redistribute it
+under certain conditions; type `show c' for details.
+

The hypothetical commands show w and show c should show the appropriate parts of the General Public License. Of course, your program’s commands might be different; for a GUI interface, you would use an “about box”.

You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see https://www.gnu.org/licenses/.

The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read https://www.gnu.org/licenses/why-not-lgpl.html.

Jungfraujoch exceptions to GPL

As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL.

We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose of simulating FPGA design on CPU.

If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch API it may be distributed under terms of your choosing without being subject to GPL requirements.

\ No newline at end of file diff --git a/NAMING.html b/NAMING.html new file mode 100644 index 00000000..8a43be19 --- /dev/null +++ b/NAMING.html @@ -0,0 +1 @@ + Naming — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Naming

The software is Swiss, and so are its names: both halves of the system are named after places in the Alps that are, in one way or another, about moving a lot of something up a steep mountain as efficiently as possible — usually by train. Throughput, in other words.

Part

Name

What it does

Streaming / acquisition

Jungfraujoch

Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing.

Data processing

Rugnux

Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the rugnux tool).

Jungfraujoch

The Jungfraujoch is a high mountain col in the Bernese Alps, the saddle (Joch is German for “yoke” or “col”) between the peaks Jungfrau and Mönch, at 3,466 m. It is the site of the High Altitude Research Station Jungfraujoch, whose long-running atmospheric measurements are co-operated by the Paul Scherrer Institute — the same institute that develops this software and the JUNGFRAU detector.

The name is also a small piece of word-play. PSI’s JUNGFRAU detector and DECTRIS’s EIGER detector are both named after Bernese Alps peaks (the famous trio is Eiger, Mönch, Jungfrau). The Jungfraujoch — the pass between Jungfrau and Mönch — is where those two detector worlds meet.

And it fits the theme of the whole project: the Jungfraujoch is reached by the Jungfraubahn, whose terminus is the highest railway station in Europe (3,454 m, the “Top of Europe”). It is the closest you can get to that summit in a genuinely high-throughput way — by train, moving crowds up the mountain — which is exactly what the streaming side of this software does with detector frames.

Pronunciation (German): Jungfraujoch ≈ YUNG-frow-yokh. “Jung” as in young, “frau” rhymes with cow, and the final “joch” ends in the guttural ch of Scottish loch or German Bach — not a hard k.

Rugnux

Piz Rugnux is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern Switzerland. (Piz is the Romansh word for “peak”.) It rises above the Albula line of the Rhaetian Railway (Rhätische Bahn), part of the “Rhaetian Railway in the Albula / Bernina Landscapes” — a UNESCO World Heritage Site (Welterbe).

That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude in very little horizontal distance, it corkscrews through a series of helical (spiral) tunnels looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a steep mountain efficiently — the same idea the data-processing side of this software is built around: pushing a large volume of diffraction data through the analysis pipeline.

So the theme is consistent — Swiss mountains, trains, and throughput — while keeping the two subsystems clearly distinct: Jungfraujoch streams, Rugnux processes.

Pronunciation (Romansh): Piz Rugnux ≈ peets roo-NYOOKS. The “gn” is a soft palatal ñ, as in canyon or Italian gnocchi, not two separate sounds.

What is Romansh?

Romansh (Rumantsch) is the fourth national language of Switzerland, alongside German, French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the canton of Graubünden. It survives in several regional idioms, brought together in a standard form called Rumantsch Grischun. Naming the processing engine with a Romansh mountain is a small nod to the least-spoken but no-less-Swiss corner of the country.

\ No newline at end of file diff --git a/OPENAPI.html b/OPENAPI.html new file mode 100644 index 00000000..6c7801be --- /dev/null +++ b/OPENAPI.html @@ -0,0 +1,2 @@ + OpenAPI — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

OpenAPI

OpenAPI specs

See document with detailed OpenAPI specs.

Python client

Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. For convenience, we provide Python client as jfjoch-client PyPi package. To install the client you can use pip tool:

pip install jfjoch-client
+

See API reference from the OpenAPI generator.

\ No newline at end of file diff --git a/OPENAPI_SPECS.html b/OPENAPI_SPECS.html new file mode 100644 index 00000000..de82642b --- /dev/null +++ b/OPENAPI_SPECS.html @@ -0,0 +1 @@ + OpenAPI specification — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

OpenAPI specification

See document with detailed OpenAPI specs generated with Redocly.

\ No newline at end of file diff --git a/PIXEL_MASK.html b/PIXEL_MASK.html new file mode 100644 index 00000000..789d777e --- /dev/null +++ b/PIXEL_MASK.html @@ -0,0 +1,13 @@ + Pixel mask — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Pixel mask

Mask format

Jungfraujoch follows generally NXmx format format for pixel mask. Pixel mask is described as 32-bit unsigned integer array of size the same as the image. Conditions to mask pixel are described by setting a particular bit to one. This way it is possible to encode reason why pixel is included in the pixel mask, also for one pixel there can be multiple reasons encoded at the same time.

Bit values are set as follows:

Bit 0 - gap (pixel with no sensor)

Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn’t set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask)

Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization)

Bit 8 - user defined mask

Bit 30 - module edge (only for PSI systems)

Bit 31 - chip edge interpolated pixel (multipixel)

Custom user mask

Jungfraujoch allows to upload custom user mask. This happens in two steps. First create mask in TIFF format:

import numpy as np
+import tifffile as tiff
+
+# Create a 2068x2164 numpy array filled with zeros, with 32-bit unsigned integers
+array = np.zeros((2068, 2164), dtype=np.uint32)
+
+# Mark the pixel (300, 400) with the value 1
+array[300, 400] = 1
+
+# Save the array as a TIFF file
+tiff.imwrite('mask.tiff', array)
+

Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8).

Then upload the mask to Jungfraujoch server:

curl -v http://<jfjoch_broker http address>/config/user_mask.tiff -XPUT --data-binary @mask.tiff
+
\ No newline at end of file diff --git a/REPOSITORIES.html b/REPOSITORIES.html new file mode 100644 index 00000000..97e3104b --- /dev/null +++ b/REPOSITORIES.html @@ -0,0 +1,5 @@ + Linux package repositories — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Linux package repositories

For convenience, we are providing package repositories. With versions including and excluding CUDA linking. We recommend to install Jungfraujoch viewer from nocuda repository and remaining packages from cuda12/cuda13 repository.

RHEL based systems

For RHEL systems we provide the following repositories:

RHEL version

CUDA

Repository file

8.x

12.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo

8.x

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo

9.x

13.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda13.repo

9.x

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-nocuda.repo

To install the repository, run:

dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo
+

Currently signing of RPMs is not supported, so the repository file needs to be manually modified to set gpgcheck=0 or installation must run with --nogpgcheck.

We provide the following packages in the repository:

  • jfjoch

  • jfjoch-driver

  • jfjoch-writer

  • jfjoch-viewer

Ubuntu based systems

For Ubuntu systems, we also provide the following repositories:

sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc
+echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+sudo apt update
+

$distribution uses Ubuntu names jammy (22.04) and noble (24.04). $component can be set to cuda13 and nocuda.

We provide the following packages in the repository:

  • jfjoch-jfjoch

  • jfjoch-driver

  • jfjoch-writer

  • jfjoch-viewer

Ubuntu packages are currently only going through a very limited testing.

\ No newline at end of file diff --git a/RUGNUX.html b/RUGNUX.html new file mode 100644 index 00000000..a9244f9f --- /dev/null +++ b/RUGNUX.html @@ -0,0 +1,10 @@ + rugnux — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

rugnux

rugnux is the offline crystallographic data-analysis tool of Jungfraujoch — the data-processing half of the system (see Naming for where the name comes from). It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the results to a _process.h5 file, plus reflection files (.mtz/.cif/.hkl) when merging is requested.

It runs the same analysis code as the online and interactive tools, just driven from the command line over a file rather than a live detector stream.

Note. rugnux is under very active development. This page describes the tool and its options at a high level; the authoritative, always-current list of options is the program’s own usage message — run rugnux with no arguments.

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

Displayed on screen (results not saved to disk)

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Use rugnux to re-analyse data after acquisition, to experiment with processing parameters, or to produce merged intensities for downstream structure solution.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. The CUDA build provides the GPU fast-feedback indexer (ffbidx) and the GPU FFT indexer (fft); without CUDA only the CPU fftw indexer is available. Spot finding, integration and scaling run on the CPU and scale with the thread count (-N).

Input and output

Input is a single Jungfraujoch HDF5 master file (NXmx-based). If the dataset already contains stored spot lists, two-pass rotation indexing can reuse them instead of re-running spot finding on the first pass.

Output (controlled by -o, --output-prefix, default output):

  • <prefix>_process.h5 — NXmx-compliant HDF5 with derived metadata (spots, indexing, integration, azimuthal integration, per-image statistics). See HDF5 / NeXus data format for the layout. Written by default only when not merging (i.e. under --no-merge); add --write-process-h5 to also write it when merging.

  • Merging is on by default (--no-merge disables it). The merged reflections are written as <prefix>.cif (mmCIF — the default), or <prefix>.mtz / <prefix>.hkl depending on --scaling-output. Both the mmCIF and the MTZ carry the refined unit cell (from rotation indexing) and the space group determined from systematic absences (constrained to the indexed lattice symmetry). No-reference scaling additionally emits per-iteration <prefix>_iterN_scale.dat.

Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. By default the written resolution is trimmed automatically where CC1/2 falls off (--resolution-cutoff cc-logistic, CC1/2 target 0.30); set --scaling-high-resolution to fix the limit by hand, or --resolution-cutoff off to keep the full range.

Re-scaling and re-merging (rugnux --scale)

The --scale mode re-scales and merges the already-integrated reflections stored in a _process.h5 file, without re-running spot finding or integration. Use it to re-merge quickly with a different space group, resolution limit, anomalous setting or reference MTZ. It reuses the same -o/-N/-s/-e/-S/-A/-B/-z/--scaling-* options as the full run, and (unlike the full pipeline) does not run a space-group search, so pass -S for the correct symmetry.

Quick start

Rotation data

Index, integrate, scale and merge a rotation sweep, fully de novo:

rugnux rotation_master.h5 \
+    -o lyso_rot -N 32 \
+    --scaling-high-resolution 1.4
+

Because the dataset carries a rotation goniometer axis, it is processed as rotation data by default: two-pass rotation indexing (index the sweep once, then process every frame against that lattice) with the rot3d partiality model (rotation partials combined into 3D fulls). Scaling and merging run by default (for both rotation and stills; --no-merge turns them off); the unit cell is taken from the rotation indexer and the space group is determined from systematic absences, and both are written into the merged .cif.

Run fully de novo (no -C/-S) for the best result — supplying a cell or space group up front tends to degrade low-symmetry cases. --scaling-high-resolution (set it to your expected resolution) sharpens both the space-group search and the error model. To tune the first pass use --two-pass-rotation=100 (or -R100 — the first-pass image count); to force the sweep to be treated as independent stills use --force-still.

After the per-frame scale-fulls step, rotation scaling applies two correction surfaces, on by default (--no-scaling-corrections disables both):

  • Decay — a global Debye–Waller relative-B over the run, for the radiation damage that weakens later frames more at high resolution (a resolution×time systematic the resolution-flat per-frame scale cannot remove). It only engages when the total relative-B exceeds a physical floor (2 Ų).

  • Absorption — a smooth multiplicative factor over the diffracted-beam direction in the goniometer frame (path length through the crystal). Negligible at hard X-rays / thin crystals; it matters at low photon energy. Its benefit shows up most on model-based metrics: a smooth absorption error largely cancels among symmetry mates (little effect on the error model / ISa) but still biases the intensities, so it measurably lowers Rfree.

Both are cross-validated — fitted on even-numbered frames and kept only if they improve the held-out odd-frame symmetry-equivalent agreement by a clear margin (and vice versa) — so where the systematic is absent they are a no-op rather than a source of added noise; that is why they are safe to leave on. They run on the GPU when one is present, at negligible cost.

Still / serial data

A dataset with no goniometer axis (e.g. a serial grid scan) is processed as independent stills automatically — no flag needed. Known-cell indexing with the GPU fast-feedback indexer, then merge against a reference structure:

rugnux serial_master.h5 \
+    -o lyso_serial -N 32 \
+    -X ffbidx -C 79,79,38,90,90,90 -S 96 \
+    --spot-sigma 4 \
+    -z reference.mtz \
+    --scaling-high-resolution 1.8
+

ffbidx requires a known cell (-C) and is the indexer of choice for sparse serial stills. For weak serial data, tightening spot finding with --spot-sigma 4 typically raises the indexing rate substantially. If a dataset does carry a goniometer axis but you want per-frame stills processing anyway, add --force-still.

Command-line options

General:

Option

Description

-o, --output-prefix <txt>

Output file prefix (default: output)

-N, --threads <num>

Number of worker threads (default: all hardware threads)

-s, --start-image <num>

First image to process (default: 0)

-e, --end-image <num>

Last image to process (default: all)

-t, --stride <num>

Process every n-th image (default: 1)

-v, --verbose

Verbose output

Modes (default: full analysis — spot finding, indexing, integration and merging):

Option

Description

--azint-only

Only run azimuthal integration (no spot finding/indexing); writes <prefix>_process.h5

--scale

Only re-scale/merge the already-integrated reflections in the input _process.h5 (no re-integration)

Spot finding:

Option

Description

--spot-sigma <num>

Noise sigma level for spot finding (default: 3.0)

--spot-threshold <num>

Photon-count threshold for spot finding (default: 10)

--spot-high-resolution <num>

High-resolution limit for spot finding, Å (default: 1.5)

--max-spots <num>

Maximum spot count (default: 250)

--detect-ice-rings[=on|off]

Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling/merging; overrides the dataset/master-file setting (default: use the dataset value)

Azimuthal integration (the radial profile behind the per-image ice-ring score):

Option

Description

-q, --azim-q-spacing <num>

Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings)

--azim-min-q <num>

Minimum Q, 1/Å

--azim-max-q <num>

Maximum Q, 1/Å

--azim-phi-bins <num>

Number of azimuthal (phi) bins (default: 1)

--polarization-correction <on|off>

Enable/disable the azimuthal polarization correction

--solid-angle-correction <on|off>

Enable/disable the azimuthal solid-angle correction

Indexing:

A dataset with a rotation goniometer axis is processed as rotation data (two-pass rotation indexing) by default; a dataset without one is processed as independent stills. --force-still overrides the former; the -R / --single-pass-rotation / --force-rotation-lattice flags request rotation explicitly and pick the pass or lattice.

Option

Description

--force-still

Treat a rotation (goniometer) dataset as independent stills instead of rotation

-X, --indexing-algorithm <txt>

FFBIDX | FFT | FFTW | Auto | None

-C, --unit-cell <cell>

Reference unit cell "a,b,c,alpha,beta,gamma" (required by ffbidx)

-S, --space-group <num>

Space group number (used for indexing and scaling)

-r, --refine <txt>

Geometry refinement: none | orientation | beam_and_lattice (default)

-R, --two-pass-rotation[=num]

Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100)

--single-pass-rotation[=num]

Online-like single-pass rotation indexing (optional min angular range, deg)

--redo-rotation-spots

Redo spot finding for the two-pass rotation first pass

--force-rotation-lattice <vec>

Force rotation lattice (9 floats, Å), skipping the first pass

Indexer choice in brief: ffbidx (GPU) refines toward a known cell and is best for sparse serial stills; fft (GPU) / fftw (CPU) index de novo and suit strong rotation data. See the CPU/GPU data-analysis reference for the algorithms.

Scaling and merging:

Option

Description

--no-merge

Skip scaling and merging (on by default); write only the per-image _process.h5

-A, --anomalous

Anomalous mode (keep Friedel pairs separate)

-B, --refine-bfactor

Refine a per-image B-factor (stills only)

--scale-fulls / --no-scale-fulls

rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills

--smooth-g[=deg]

rot3d: smooth the per-frame scale G over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off)

--no-scaling-corrections

rot3d: disable the default-on decay + absorption correction surfaces fitted on the fulls after scale-fulls (see below)

--capture-uncertainty <num>

rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise)

--min-captured-fraction <num>

rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off)

--scaling-high-resolution <num>

High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below)

--resolution-cutoff <txt>

Automatic high-resolution cutoff for the written reflections and reported shells: cc-logistic | off (default: cc-logistic; ignored when --scaling-high-resolution is set)

--resolution-cc-target <num>

CC1/2 target defining the cc-logistic fall-off (default: 0.30)

--resolution-shells <num>

Number of resolution shells in the reported statistics table (default: 10)

--min-partiality <num>

Minimum partiality to accept a reflection (default: 0.02)

--reject-outliers <num>

Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rot3d, off otherwise)

--reject-delta-cchalf <num>

Drop images with ΔCC1/2 below mean − N·stddev (default: off)

--min-image-cc <num>

Per-image CC limit, percent (default: no limit)

--mosaicity <num>

Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed

--scaling-iterations <num>

Scaling iterations with no reference data (default: 3)

--scaling-output <txt>

Reflection output format: cif (mmCIF, default) | mtz | txt

-z, --reference-mtz <file>

Reference MTZ (enables reference-driven scaling)

--reference-column <label>

Reference MTZ column to use (default: auto — F-model, else IMEAN/I/…)

--write-process-h5

Also write the (large) _process.h5 when merging (default: only .mtz/.cif)

Integration:

Option

Description

--integrator <txt>

Spot integrator: gaussian (profile-fit, default) | empirical | boxsum (classical fallback)

--integration-radius <r>

Signal-box radius r1, or r1,r2,r3 (px). One value ⇒ r2=r1+2, r3=r1+4

--bandwidth <num>

Relative X-ray bandwidth FWHM (e.g. 0.01 for a 1% DMM); default from file or 0 (monochromatic)

Geometry overrides (defaults are taken from the input file; override them to reprocess with a corrected geometry):

Option

Description

--beam-x <num>

Beam centre X (pixel)

--beam-y <num>

Beam centre Y (pixel)

--detector-distance <num>

Detector distance (mm)

--wavelength <num>

Wavelength (Å)

--rot1 <num>

PONI detector rotation 1 (rad)

--rot2 <num>

PONI detector rotation 2 (rad)

--polarization <num>

Polarization factor

\ No newline at end of file diff --git a/SOFTWARE.html b/SOFTWARE.html new file mode 100644 index 00000000..4365af9f --- /dev/null +++ b/SOFTWARE.html @@ -0,0 +1 @@ + Software requirements — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Software requirements

Operating system

Recommended operating system is Red Hat Enterprise Linux (RHEL) / Rocky Linux versions 8 or 9. For this operating systems we provide RPMs with pre-built binaries to simplify deployment. On experimental basis we also build repositories for Ubuntu 22.04 and 24.04.

Running Jungfraujoch on Red Hat Enterprise Linux 7 is currently not tested and not recommended, but likely possible with providing some packages from external repositories.

The desktop viewer jfjoch_viewer (only) can additionally be built on Windows 11 with Visual Studio 2026 (MSVC), CUDA 13.3 and Qt 6.11 — see jfjoch_viewer ▸ Building from source on Windows. The Windows installer bundles the Qt runtime, and on the CUDA build the CUDA runtime (cuFFT) as well, so end users need neither Qt nor a CUDA toolkit installed — only an NVIDIA GPU driver for the GPU path. The rest of Jungfraujoch is Linux-only.

Software dependencies

Required:

  • C++20 compiler and C++20 standard library; recommended GCC 11+ or clang 14+ (Intel OneAPI, AMD AOCC)

  • CMake version 3.26 or newer + a build tool (GNU make or Ninja)

  • zlib compression library

  • Eigen (header-only linear algebra library), version 3.4.x (the build requests Eigen3 3.4, which Eigen’s same-major-version rule does not satisfy with 5.x)

HDF5, libtiff and libjpeg-turbo used to be required system packages; they are now downloaded and built automatically by CMake (see the note below), so they no longer need to be installed.

Optional:

  • CUDA compiler version 12.8 or newer - required for the MX fast feedback indexer and GPU analysis

  • FFTW library - for indexing if GPU/CUDA is absent (also auto-downloaded by CMake)

  • Node.js - to build the frontend

  • Qt version 6 (for jfjoch_viewer)

Many further dependencies (spdlog, Zstandard, HDF5, slsDetectorPackage, libzmq, libtiff, libjpeg-turbo, Ceres, the fast feedback indexer, Catch2, …) are downloaded automatically by CMake and statically linked; building therefore requires network access on the first configure. zlib and Eigen are the exception — they must be preinstalled (found via find_package); on Windows, where they are not present system-wide, install them into a prefix and point CMAKE_PREFIX_PATH at it (see Building from source on Windows). Others are vendored directly in the source tree. The complete list of third-party components, with copyright holders, licenses and verbatim license texts, is in Third-party software notices and the licenses/ directory.

\ No newline at end of file diff --git a/SOFTWARE_INTEGRATION.html b/SOFTWARE_INTEGRATION.html new file mode 100644 index 00000000..51285ace --- /dev/null +++ b/SOFTWARE_INTEGRATION.html @@ -0,0 +1,2 @@ + Integration with MX data processing software — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Integration with MX data processing software

XDS

Jungfraujoch files are compatible with XDS, but there is a need of a dedicated plugin. First we recommend to use Jungfraujoch own XDS plugin. It is available for Linux only and can be downloaded from Gitea release directory (compiled on RHEL 8), it is also distributed in jfjoch_viewer RPM/APT packages. To use the plugin, download the file libjfjoch_xds_plugin.so.1.0.0 (three numbers at the end represent version of the plugin, and can differ later in time), save it to common directory (e.g., /opt/xds) and add the following line in the XDS.INP file:

LIB="/opt/xds/libjfjoch_xds_plugin.so.1.0.0"
+

We are also testing XDS with Durin and Neggia plugins, though they don’t have full functionality:

  • Neggia plugin doesn’t support HDF5 virtual data sets. It can be downloaded from github.com/dectris/neggia.

  • Durin has known bugs with handling non-DECTRIS files (so with virtual data sets or single format HDF5 file format). We recommend Durin plugin prepared by the Global Phasing consortium: github.com/CV-GPhL/durin, rather than original from the Diamond Light Source.

DIALS

Jungfraujoch files are tested regularly with DIALS (currently v. 3.27.0) xia2.ssx pipeline for serial crystallography. There is one known limitation: files generated with NXmxLegacy format (mimicking DECTRIS filewriter1 format) are not handled properly with DIALS. VDS based HDF5 format (NXmxVDS) is recommended, when using DIALS.

CrystFEL

Jungfraujoch files are compatible with CrystFEL.

\ No newline at end of file diff --git a/TESTS.html b/TESTS.html new file mode 100644 index 00000000..da73ce23 --- /dev/null +++ b/TESTS.html @@ -0,0 +1 @@ + Tests — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Tests

Automated test routine is then accessible as tests/jfjoch_test. There are also benchmark routines:

  • jfjoch_hdf5_test to measure HDF5 dataset writing speed (single threaded)

  • jfjoch_offline_process to apply spot finding and indexing routines in Jungfraujoch to an example dataset - this is equivalent to FPGA spot finding algorithm, but NOT performance equivalent as it is particularly not-efficient

  • jfjoch_fpga_test to test quality/performance of FPGA card(s) and software routines

In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin, XDS Neggia plygin and CrystFEL. Input files for these programs are placed in xds_durin, xds_neggia and crystfel folders. See .gitlab-ci.yml for details.

\ No newline at end of file diff --git a/THIRD_PARTY_NOTICES.html b/THIRD_PARTY_NOTICES.html new file mode 100644 index 00000000..84153fb8 --- /dev/null +++ b/THIRD_PARTY_NOTICES.html @@ -0,0 +1,2 @@ + Third-party software notices — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Third-party software notices

Jungfraujoch is licensed under GPL-3.0 (see LICENSE); the FPGA design is licensed under CERN-OHL-S-2.0 (see fpga/LICENSE). It builds on a number of third-party components, acknowledged below as required by their licenses.

This file is the human-readable manifest. The verbatim license texts live in the licenses/ directory (regenerate with bash licenses/COLLECT.sh). The frontend’s bundled JavaScript dependencies are listed separately in frontend/dist/THIRD_PARTY_LICENSES.txt, generated at build time (npm run licenses).

All licenses below are GPL-3.0-compatible.

Fetched at build time and statically linked into the C++ binaries

These are downloaded by CMake (FetchContent / ExternalProject) during the first configure and linked into the Jungfraujoch executables.

Component

Version

Copyright

License (SPDX)

License text

spdlog

1.17.0

Gabi Melman

MIT

spdlog.txt

Zstandard

(pinned)

Meta Platforms, Inc.

BSD-3-Clause

zstd.txt

HDF5

2.1.0

The HDF Group; UIUC

BSD-3-Clause-style

hdf5.txt

slsDetectorPackage

8.0.2 / 9.2.0

PSI

LGPL-3.0-or-later

LGPL, GPL

cpp-httplib

0.39.0

Yuji Hirose

MIT

cpp-httplib.txt

libzmq (ZeroMQ)

4.3.5

iMatix and contributors

MPL-2.0

libzmq.txt

libtiff

4.7.1

Sam Leffler; SGI

libtiff (BSD-like)

libtiff.txt

FFTW

3.3.10

Matteo Frigo; MIT

GPL-2.0-or-later

fftw.txt

Ceres Solver

(pinned)

Google Inc. and contributors

BSD-3-Clause

ceres-solver.txt

fast-feedback-indexer

(pinned)

PSI

BSD-3-Clause

fast-feedback-indexer.txt

libjpeg-turbo

(pinned)

D. R. Commander and others; IJG

IJG + BSD-3-Clause + Zlib

libjpeg-turbo.txt

Catch2

3.13.0

Catch2 Authors

BSL-1.0

catch2.txt

Catch2 is used only to build the test binary (jfjoch_test) and is not part of any shipped artifact; it is listed here for completeness.

Vendored directly in the repository

These are copied into the source tree (see the path) rather than fetched.

Component

Path

Copyright

License (SPDX)

License text

nlohmann/json

include/nlohmann/

Niels Lohmann

MIT

nlohmann-json.txt

Macaron Base64

include/base64/

tomykaira

MIT

base64-macaron.txt

TinyCBOR

frame_serialize/tinycbor/

Intel Corporation

MIT

tinycbor.txt

Bitshuffle

compression/bitshuffle/

Kiyoshi Masui

MIT

bitshuffle.txt

Bitshuffle (h-perf)

compression/bitshuffle_hperf/

Kal Cutter (DECTRIS)

Apache-2.0

bitshuffle-hperf.txt

LZ4

compression/lz4/

Yann Collet

BSD-2-Clause

lz4.txt

HLS arbitrary-precision types

fpga/include/

Xilinx, Inc.

Apache-2.0

xilinx-hls-headers.txt

GEMMI

gemmi_gph/

Global Phasing Ltd.

MPL-2.0

gemmi.txt

xbflash.qspi

tools/xbflash.qspi/

Xilinx / AMD

Apache-2.0

xbflash-qspi.txt

wingetopt

tools/wingetopt/

Todd C. Miller; The NetBSD Foundation

ISC AND BSD-2-Clause

wingetopt.txt

Runtime libraries and SDKs (shipped in binaries, not in the source tree)

Component

Used by

License

Notice

Qt 6

jfjoch_viewer

LGPL-3.0

notice, LGPL-3.0

NVIDIA CUDA Toolkit (cudart, cuFFT)

CUDA builds

NVIDIA CUDA EULA

notice, EULA

zlib

everywhere (compression)

Zlib

zlib.txt

Eigen

analysis libs, Ceres, ffbidx (header-only)

MPL-2.0 (+ BSD parts)

eigen.txt, README

Frontend (npm) dependencies

The React/TypeScript frontend (frontend/) bundles a large transitive tree of npm packages, overwhelmingly MIT/ISC/BSD/Apache-2.0 licensed. Their full notices are generated automatically:

cd frontend && npm run licenses     # writes dist/THIRD_PARTY_LICENSES.txt
+

The generated file is produced as part of the frontend build target and installed alongside the served frontend, so the shipped web UI carries its own attribution.

Notes on weak-copyleft and attribution-sensitive components

  • MPL-2.0 (Eigen, GEMMI, libzmq): file-level copyleft. GEMMI is vendored in gemmi_gph/ in trimmed form; libzmq is fetched at build time; Eigen is provided externally (header-only). The corresponding source is available from each project upstream.

  • FFTW is GPL-2.0-or-later — compatible with, and absorbed by, this project’s GPL-3.0 license.

  • Apache-2.0 components: where upstream ships a NOTICE file, it is reproduced in the corresponding licenses/ text.

  • Qt (LGPL-3.0) and NVIDIA CUDA (EULA) carry redistribution conditions beyond a copyright notice; see their dedicated notice files. The verbatim LGPL-3.0 and CUDA EULA texts are bundled (licenses/Qt6-LGPL-3.0.txt, licenses/NVIDIA-CUDA-EULA.txt); the CUDA EULA is the one shipped with CUDA Toolkit 12.8 — replace it if you build against a different toolkit version.

\ No newline at end of file diff --git a/TOOLS.html b/TOOLS.html new file mode 100644 index 00000000..cb7e1129 --- /dev/null +++ b/TOOLS.html @@ -0,0 +1,13 @@ + Tools — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Tools

Besides the main services (jfjoch_broker, jfjoch_writer, jfjoch_viewer), the repository ships a number of command-line tools. Each prints its own usage when run with -h or without arguments.

Data analysis

rugnux

Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, integration, scaling/merging) on a stored HDF5 dataset, producing a _process.h5 file and, when merging, reflection files. Merging is on by default (--no-merge disables it). Two extra modes narrow the work: --azint-only runs only azimuthal integration (no spot finding/indexing), and --scale re-scales/merges the already-integrated reflections in a _process.h5 without re-integrating. See rugnux.

jfjoch_extract_hkl

Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across neighbouring images and compare against an XDS INTEGRATE.HKL reference.

FPGA / PCIe card management

jfjoch_pcie_status

Prints detailed status information about the card. Safe to run during data collection:

./jfjoch_pcie_status /dev/jfjoch0
+

jfjoch_pcie_net_cfg

Reads and modifies the network configuration of the card’s interfaces:

jfjoch_pcie_net_cfg <device name>
+     Read configuration for all network interfaces of a device
+jfjoch_pcie_net_cfg <device name> <if number>|fgen
+     Read configuration for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen ipv4 <IPv4 address>
+     Set IPv4 address for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen direct 0|1
+     Set direct mode for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen clear
+     Clear Ethernet counters for a particular network interface / internal frame generator
+

jfjoch_pcie_clear_net_counters

Resets the card’s Ethernet, UDP and ICMP packet counters (which otherwise run from power-on):

./jfjoch_pcie_clear_net_counters /dev/jfjoch0
+

Testing, benchmarking and simulation

jfjoch_udp_simulator

UDP packet simulator used to test the Jungfraujoch FPGA receiver.

jfjoch_fpga_test

Exercises and benchmarks the FPGA data path and receiver. With -H it runs the high-level synthesis C model on the CPU, so no FPGA device is required.

jfjoch_lite_perf_test

Performance test of the lite (CPU/GPU) analysis path — indexing, integration and optional file writing.

jfjoch_hdf5_test

Tests single-threaded HDF5 writer performance.

jfjoch_simplon_test

Minimal test client for a DECTRIS SIMPLON detector API.

\ No newline at end of file diff --git a/VERSIONING.html b/VERSIONING.html new file mode 100644 index 00000000..188b13c6 --- /dev/null +++ b/VERSIONING.html @@ -0,0 +1 @@ + Semantic versioning — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Semantic versioning

Jungfraujoch is following semantic versioning. For this purpose we define public API as following:

  • OpenAPI configuration interface

  • CBOR serialization ZeroMQ stream

  • HDF5 file format

This means that changes in the format of thereof must be accompanied by version change - major version in case of breaking changes, minor version in case of feature expansion.

NOTE: FPGA design, PCIe driver, and internal libraries are not part of the public API and are considered internals of Jungfraujoch. Breaking changes in these components can happen without incrementing major version of the whole package. It will be marked in changelog.

\ No newline at end of file diff --git a/WEB_FRONTEND.html b/WEB_FRONTEND.html new file mode 100644 index 00000000..3b246f6c --- /dev/null +++ b/WEB_FRONTEND.html @@ -0,0 +1 @@ + Web frontend — Jungfraujoch 1.0.0-rc.158 documentation Skip to content

Web frontend

Jungfraujoch is equipped with React-based web frontend for user-friendly experience. Frontend has the following options:

  • Presenting current state of the detector

  • Plotting results of online quality calculations

  • Showing live view images from the detector

  • JUNGFRAU calibration numbers

  • Configuring the detector, as well as pedestal/initialization operations

Frontend is written in TypeScript. For details see frontend/ directory.

\ No newline at end of file diff --git a/_downloads/625caea52ebbd47be3d6ed9efa4ee972/redoc-static.html b/_downloads/625caea52ebbd47be3d6ed9efa4ee972/redoc-static.html new file mode 100644 index 00000000..e4df3b4d --- /dev/null +++ b/_downloads/625caea52ebbd47be3d6ed9efa4ee972/redoc-static.html @@ -0,0 +1,964 @@ + + + + + + Jungfraujoch + + + + + + + + + +
API docs by Redocly

Jungfraujoch (1.0.0-rc.158)

Download OpenAPI specification:

Filip Leonarski (Paul Scherrer Institute): filip.leonarski@psi.ch License: GPL-3.0

API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). +Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. +Jungfraujoch uses FPGA boards to acquire data at high data rates.

+

License Clarification

While this API definition is licensed under GPL-3.0, the GPL copyleft provisions do not apply +when this file is used solely to generate OpenAPI clients or when implementing applications that +interact with the API. Generated client code and applications using this API definition are not +subject to the GPL license requirements and may be distributed under terms of your choosing.

+

This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and +the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that +user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not +derivative works of the kernel and are not subject to the terms of the GPL.

+

This exception is intended to allow wider use of this API specification without imposing GPL +requirements on applications that merely interact with the API, regardless of whether they +communicate through network calls or other mechanisms.

+

Initialize detector and data acquisition

Should be used in two cases:

+
    +
  • Detector is in Inactive state
    • +
  • Detector is in Error state +X-ray shutter must be closed. +This operation will reconfigure network interface of the detector. +During operation of the detector it is recommended to use the POST /pedestal operation instead. +If storage cells are used, the execution time might be few minutes.
    • +
+

This is async function - one needs to use POST /wait_till_done to ensure operation is done.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Collect dark current for the detector

Updates calibration of the JUNGFRAU detector. Must be in Idle state.

+

X-ray shutter must be closed. Recommended to run once per hour for long integration times (> 100 us).

+

This is async function - one needs to use POST /wait_till_done to ensure operation is done.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Start detector

Start data acquisition. +Detector must be in Idle state. +Default behavior is for the call to block until detector is ready to accept soft/TTL triggers. +However, this behavior can be changed by settings async_start to true in the request body, +in which case the call will return immediately and one needs to use /wait_until_running to ensure detector is ready to run.

+
Request Body schema: application/json
images_per_trigger
integer <int64> >= 1
Default: 1

For standard synchrotron data collection - this is number of images collected per one TTL trigger +For XFEL (pulsed source) - this number is ignored and set to 1 +For storage cell mode - this number is ignored and set to number of storage cells

+
ntrigger
integer <int64> >= 1
Default: 1

Number of TTL trigger that the detector is expected to receive during data collection

+
image_time_us
integer <int64> >= 0

Image time. +If not provided (or zero value) the frame time is assumed as default. +For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time.
In XFEL mode: summation happens for frames collected with multiple triggers. +Ignored for storage cells and if raw data are saved.

+
beam_x_pxl
required
number <float>

/entry/detector/beam_center_x in NXmx +Beam center in X direction [pixels]

+
beam_y_pxl
required
number <float>

/entry/detector/beam_center_y in NXmx +Beam center in X direction [pixels]

+
detector_distance_mm
required
number <float> >= 0

/entry/detector/distance in NXmx Detector distance [mm]

+
incident_energy_keV
required
number <float> [ 0.001 .. 500 ]

Used to calculate /entry/beam/incident_wavelength in NXmx +Incident particle (photon, electron) energy in keV

+
file_prefix
string
Default: ""

Prefix for filenames. If left empty, no file will be saved.

+
images_per_file
integer <int64> >= 0
Default: 1000

Number of files in a single HDF5 data file (0 = write all images to a single data file).

+
space_group_number
integer <int64> [ 1 .. 194 ]

Number of space group for the crystal. Currently used solely as metadata, not relevant for image processing done in Jungfraujoch.

+
sample_name
string
Default: ""

/entry/sample/name in NXmx +Sample name

+
compression
string
Default: "bslz4"
Enum: "bslz4" "bszstd" "bszstd_rle" "bszstd_rlehuf" "none"

Compression type for the images transferred over ZeroMQ and saved to HDF5 file.

+
total_flux
number <float>

/entry/beam/total_flux in NXmx +Flux incident on beam plane in photons per second. In other words this is the flux integrated over area. [photons/s]

+
transmission
number <float> [ 0 .. 1 ]

/entry/instrument/attenuator/attenuator_transmission +Transmission of attenuator (filter) [no units]

+
object (rotation_axis)

Definition of a crystal rotation axis

+
object (grid_scan)

Definition of a grid scan (mutually exclusive with rotation_axis)

+
header_appendix
any

Header appendix, added as user_data/user to start ZeroMQ message (can be any valid JSON) +In general, it is not saved in HDF5 file.

+

However, if values are placed in "hdf5" object, jfjoch_writer will write them in /entry/user of the HDF5 file. +This applies solely to string and number (double floating-point). No arrays/sub-objects is allowed. +For example {"hdf5": {"val1":1, "val2":"xyz"}}, will write /entry/user/val1 and /entry/user/val2.

+
image_appendix
any

Image appendix, added as user_data to image ZeroMQ message (can be any valid JSON) +Not saved in HDF5 file

+
data_reduction_factor_serialmx
number <float> [ 0 .. 1 ]
Default: 1

Rate at which non-indexed images are accepted to be forwarded to writer. +Value of 1.0 (default) means that all images are written. +Values below zero mean that non-indexed images will be accepted with a given probability.

+
pixel_value_low_threshold
integer <int64> >= 0

Set all counts lower than the value to zero. +When the value is set, negative numbers other than error pixel value are always set to zero. +Setting to zero is equivalent to turning the option off.

+
run_number
integer <int64> >= 0

Number of run within an experimental session. +Transferred over CBOR stream as "series ID", though not saved in HDF5 file. +It is highly recommended to keep this number unique for each data collection during experimental series. +If not provided, the number will be automatically incremented.

+
run_name
string

Unique ID of run. +Transferred over CBOR stream as "unique series ID", though not saved in HDF5 file. +It is highly recommended to keep this name unique for each data collection during experimental series. +If not provided, the name will be automatically generated as number + colon + file_prefix.

+
experiment_group
string

Name of group owning the data (e.g. p-group or proposal number). +Transferred over CBOR stream, though not saved in HDF5 file.

+
poisson_compression
integer <int64> [ 0 .. 16 ]

Enable lossy compression of pixel values that preserves Poisson statistics. +Requires to provide a numerical factor SQ. +Pixel value P will be transformed to round(sqrt(P) * SQ), with rounding to the closest integer. +Compression is turned off if the value is missing or it is set to zero.

+
write_nxmx_hdf5_master
boolean
Default: true

Write NXmx formatted HDF5 master file. Recommended to use for macromolecular crystallography experiments +and to turn off for other experiments.

+
save_calibration
boolean

Forward image calibration (at the moment pedestal and pedestal RMS for JUNGFRAU) using the ZeroMQ stream to writer. +If parameter is not provided calibration will be saved only if more than 4 images are recorded.

+
polarization_factor
number <float> [ -1 .. 1 ]

Polarization factor for integration; 1.0 is horizontal polarization; -1.0 is vertical polarization

+
ring_current_mA
number <float> >= 0

Ring current at the beginning of the data collection

+
sample_temperature_K
number <float> >= 0

Sample temperature in Kelvin

+
poni_rot1_rad
number <float> [ -6.28318530718 .. 6.28318530718 ]
Default: 0

PONI angle rot1 (see PyFAI documentation for details) in radians

+
poni_rot2_rad
number <float> [ -6.28318530718 .. 6.28318530718 ]
Default: 0

PONI angle rot2 (see PyFAI documentation for details) in radians

+
poni_rot3_rad
number <float> [ -6.28318530718 .. 6.28318530718 ]
Default: 0

PONI angle rot3 (see PyFAI documentation for details) in radians

+
object (unit_cell)

Unit cell parameters. Necessary to run indexing. Units of angstrom and degree

+
spot_finding
boolean
Default: true

Enable spot finding and save spots

+
object

Geometry of Smargon goniometer at SLS 2.0 / MX beamlines. +Assuming that Smargon is used as static positioner and not moving during the scan, allowing to reconstruct geometry.

+
max_spot_count
integer [ 10 .. 2000 ]
Default: 250

Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected

+
detect_ice_rings
boolean

Flag spots as ice rings and reduce their effect on indexing

+
async_start
boolean
Default: false

When set to true, /start will not wait for detector and Jungfraujoch to be ready for the measurement.

+
object

Responses

Request samples

Content type
application/json
{
  • "images_per_trigger": 1,
  • "ntrigger": 1,
  • "image_time_us": 0,
  • "beam_x_pxl": 0.1,
  • "beam_y_pxl": 0.1,
  • "detector_distance_mm": 0.1,
  • "incident_energy_keV": 0.001,
  • "file_prefix": "",
  • "images_per_file": 1000,
  • "space_group_number": 1,
  • "sample_name": "",
  • "compression": "bslz4",
  • "total_flux": 0.1,
  • "transmission": 1,
  • "goniometer": {
    },
  • "grid_scan": {
    },
  • "header_appendix": null,
  • "image_appendix": null,
  • "data_reduction_factor_serialmx": 1,
  • "pixel_value_low_threshold": 0,
  • "run_number": 0,
  • "run_name": "string",
  • "experiment_group": "string",
  • "poisson_compression": 16,
  • "write_nxmx_hdf5_master": true,
  • "save_calibration": true,
  • "polarization_factor": -1,
  • "ring_current_mA": 0.1,
  • "sample_temperature_K": 0.1,
  • "poni_rot1_rad": 0,
  • "poni_rot2_rad": 0,
  • "poni_rot3_rad": 0,
  • "unit_cell": {
    },
  • "spot_finding": true,
  • "smargon": {
    },
  • "max_spot_count": 250,
  • "detect_ice_rings": true,
  • "async_start": false,
  • "xray_fluorescence_spectrum": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition running

Block execution of external script till detector and Jungfraujoch are ready to collect data. +To not block web server for a indefinite period of time, the procedure is provided with a timeout. +Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection.

+
query Parameters
timeout
integer [ 0 .. 3600 ]
Default: 60

Timeout in seconds (0 == immediate response)

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition done

Block execution of external script till initialization, data collection or pedestal is finished. +Running this command does not affect (cancel) running data collection, it is only to ensure synchronous execution of other software.

+

To not block web server for a indefinite period of time, the procedure is provided with a timeout. +Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection.

+
query Parameters
timeout
integer [ 0 .. 3600 ]
Default: 60

Timeout in seconds (0 == immediate response)

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Send soft trigger to the detector

Generate soft trigger

+

Responses

Cancel running data collection

Command will inform FPGA network card to stop pedestal or data collection at the current stage. +Any frame that is currently being processed by CPU will be finished and sent to writer. +Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to Idle state.

+

If data collection is not running, the command has no effect.

+

Responses

Prepare detector to turn off

Should be in Idle or Error state. +Command deactivates data acquisition and turns off detector high voltage and ASIC. +Should be used always before turning off power from the detector.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Change detector configuration

Detector settings are ones that have effect on calibration, i.e., pedestal has to be collected again after changing these settings. +This can only be done when detector is Idle, Error or Inactive states. +If detector is in Idle state , pedestal procedure will be executed automatically - there must be no X-rays on the detector during the operation. +If detector is in Inactive or Error states, new settings will be saved, but no calibration will be executed.

+
Request Body schema: application/json
frame_time_us
required
integer <int64> >= 1

Interval between consecutive frames. +This is internal frame time for the JUNGFRAU detector, image time has to be integer multiply of this number. +For EIGER detector this is default frame time, not used otherwise

+
count_time_us
integer <int64>

Integration time of the detector. +If not provided count time will be set to maximum value for a given frame time.

+
internal_frame_generator
boolean
Default: false

Use internal frame generator in FPGA instead of getting data from a real detector

+
internal_frame_generator_images
integer <int64> [ 1 .. 64 ]
Default: 1

Number of images stored in the internal frame generator.

+
detector_trigger_delay_ns
integer <int64> >= 0
Default: 0

Delay between TTL trigger and acquisition start [ns]

+
timing
string (detector_timing)
Default: "trigger"
Enum: "auto" "trigger" "burst" "gated"
eiger_threshold_keV
number <float> [ 1 .. 100 ]

Threshold for the PSI EIGER detector and all DECTRIS detectors. +If value is provided, it will be used for all subsequent acquisitions, irrespective of beam energy. +If value is not provided, threshold will be determined on start of acquisition as half of incident energy. +This might lead to increased start time.

+
eiger_bit_depth
integer <int64>
Enum: 8 16 32

Bit depth of PSI EIGER read-out. This is
If value is not provided, depth will be determined based on the image time:

+
    +
  • Exposure time < 500 microseconds depth of 8 bit will be used,
    • +
  • 500 <= exposure time < 2622 microseconds depth of 16 bit will be used
    • +
  • Exposure time >= 2622 microseconds depth of 32 bit will be used.
    • +
+
jungfrau_pedestal_g0_frames
integer <int64> >= 0
Default: 2000
jungfrau_pedestal_g1_frames
integer <int64> >= 0
Default: 300
jungfrau_pedestal_g2_frames
integer <int64> >= 0
Default: 300
jungfrau_pedestal_min_image_count
integer <int64> >= 32
Default: 128

Minimum number of collected images for pedestal to consider it viable

+
jungfrau_storage_cell_count
integer <int64> [ 1 .. 16 ]
Default: 1
jungfrau_storage_cell_delay_ns
integer <int64> >= 2100
Default: 5000

Delay between two storage cells [ns]

+
jungfrau_fixed_gain_g1
boolean
Default: false

Fix gain to G1 (can be useful for storage cells)

+
jungfrau_use_gain_hg0
boolean
Default: false

Use high G0 (for low energy applications)

+

Responses

Request samples

Content type
application/json
{
  • "frame_time_us": 1,
  • "count_time_us": 0,
  • "internal_frame_generator": false,
  • "internal_frame_generator_images": 1,
  • "detector_trigger_delay_ns": 0,
  • "timing": "auto",
  • "eiger_threshold_keV": 1,
  • "eiger_bit_depth": 8,
  • "jungfrau_pedestal_g0_frames": 2000,
  • "jungfrau_pedestal_g1_frames": 300,
  • "jungfrau_pedestal_g2_frames": 300,
  • "jungfrau_pedestal_min_image_count": 128,
  • "jungfrau_storage_cell_count": 1,
  • "jungfrau_storage_cell_delay_ns": 5000,
  • "jungfrau_fixed_gain_g1": false,
  • "jungfrau_use_gain_hg0": false
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get detector configuration

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "frame_time_us": 1,
  • "count_time_us": 0,
  • "internal_frame_generator": false,
  • "internal_frame_generator_images": 1,
  • "detector_trigger_delay_ns": 0,
  • "timing": "auto",
  • "eiger_threshold_keV": 1,
  • "eiger_bit_depth": 8,
  • "jungfrau_pedestal_g0_frames": 2000,
  • "jungfrau_pedestal_g1_frames": 300,
  • "jungfrau_pedestal_g2_frames": 300,
  • "jungfrau_pedestal_min_image_count": 128,
  • "jungfrau_storage_cell_count": 1,
  • "jungfrau_storage_cell_delay_ns": 5000,
  • "jungfrau_fixed_gain_g1": false,
  • "jungfrau_use_gain_hg0": false
}

Change indexing algorithm settings

This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
algorithm
required
string (indexing_algorithm)
Default: "FFBIDX"
Enum: "FFBIDX" "FFT" "FFTW" "Auto" "None"

Selection of an indexing algorithm used by Jungfraujoch

+
fft_max_unit_cell_A
required
number <float> [ 50 .. 500 ]
Default: 250

Largest unit cell to be indexed by FFT algorithm; parameter value affects execution time of FFT

+
fft_min_unit_cell_A
required
number <float> [ 5 .. 40 ]
Default: 10

Smallest unit cell to be indexed by FFT algorithm; parameter value affects execution time of FFT

+
fft_high_resolution_A
required
number <float> [ 0.5 .. 6 ]
Default: 2

Highest resolution of spots used for FFT algorithm; parameter value affects execution time of FFT. +There is also correlation between smallest unit cell and max resolution, which need to be checked for very small systems.

+
fft_num_vectors
required
integer <int64> >= 128
Default: 16384

Number of search directions for the FFT algorithm; parameter value affects execution time of FFT.

+
tolerance
required
number <float> [ 0 .. 0.5 ]

Acceptance tolerance for spots after the indexing run - the larger the number, the more spots will be accepted

+
thread_count
required
integer <int64> [ 1 .. 64 ]

Thread count for indexing algorithm

+
geom_refinement_algorithm
required
string (geom_refinement_algorithm)
Enum: "BeamCenter" "OrientationOnly" "None"

Selection of an post-indexing least-square diffraction geometry refinement algorithm used by Jungfraujoch. +BeamCenter - This option is refining both beam center and lattice (restricted to a chosen/detected Bravais lattice). +OrientationOnly - This option is refining only orientation of the lattice.

+
unit_cell_dist_tolerance
required
number <float> [ 0.0001 .. 0.2001 ]
Default: 0.05

Relative distance tolerance for unit cell vs. reference; Lattices outside given tolerance will be ignored

+
viable_cell_min_spots
required
integer <int64> >= 5
Default: 10

Minimum number of indexed spots required for a cell to be considered viable

+
index_ice_rings
required
boolean
Default: false

Include spots marked as ice rings in the indexing run. +If dataset_settings doesn't have detect_ice_rings on, this option will have no effect on processing.

+
rotation_indexing
required
boolean
Default: false
rotation_indexing_min_angular_range_deg
required
number <float> >= 1
Default: 20
rotation_indexing_angular_stride_deg
required
number <float> >= 0
Default: 0.5
blocking
required
boolean
Default: true

Indexing in Jungfraujoch goes with a dedicated thread pool. +If set to false, the thread pool is non-blocking, i.e. if there are no threads available, image indexing will be skipped. This option is recommended for real-time processing at high frame rates. +If set to true, the thread pool will block until a thread is available.

+

Responses

Request samples

Content type
application/json
{
  • "algorithm": "FFBIDX",
  • "fft_max_unit_cell_A": 250,
  • "fft_min_unit_cell_A": 10,
  • "fft_high_resolution_A": 2,
  • "fft_num_vectors": 16384,
  • "tolerance": 0.5,
  • "thread_count": 1,
  • "geom_refinement_algorithm": "BeamCenter",
  • "unit_cell_dist_tolerance": 0.05,
  • "viable_cell_min_spots": 10,
  • "index_ice_rings": false,
  • "rotation_indexing": false,
  • "rotation_indexing_min_angular_range_deg": 20,
  • "rotation_indexing_angular_stride_deg": 0.5,
  • "blocking": true
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get indexing configuration

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "algorithm": "FFBIDX",
  • "fft_max_unit_cell_A": 250,
  • "fft_min_unit_cell_A": 10,
  • "fft_high_resolution_A": 2,
  • "fft_num_vectors": 16384,
  • "tolerance": 0.5,
  • "thread_count": 1,
  • "geom_refinement_algorithm": "BeamCenter",
  • "unit_cell_dist_tolerance": 0.05,
  • "viable_cell_min_spots": 10,
  • "index_ice_rings": false,
  • "rotation_indexing": false,
  • "rotation_indexing_min_angular_range_deg": 20,
  • "rotation_indexing_angular_stride_deg": 0.5,
  • "blocking": true
}

Change Bragg integration settings

This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
integration_model
required
string (integration_model)
Default: "ProfileGaussian"
Enum: "ProfileGaussian" "ProfileEmpirical" "BoxSum"

Bragg spot integration model. +ProfileGaussian - profile fit with a measured-width Gaussian (Kabsch-style), the default; more + accurate intensities than box summation. +ProfileEmpirical - profile fit with a per-resolution-shell empirical profile learned from strong spots. +BoxSum - classical uniform box summation minus a ring-mean background; the simpler, faster fallback.

+

Responses

Request samples

Content type
application/json
{
  • "integration_model": "ProfileGaussian"
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get Bragg integration configuration

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "integration_model": "ProfileGaussian"
}

Change file writer settings

This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
overwrite
boolean
Default: false

Inform jfjoch_write to overwrite existing files. Otherwise files would be saved with .h5.{timestamp}.tmp suffix.

+
format
string (file_writer_format)
Default: "NXmxLegacy"
Enum: "NXmxOnlyData" "NXmxLegacy" "NXmxVDS" "NXmxIntegrated" "CBF" "TIFF" "NoFileWritten"

NoFileWritten - no files are written at all +NXmxOnlyData - only data files are written, no master file +NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling +NXmxIntegrated - single HDF5 per dataset +CBF - CBF format (limited metadata) +TIFF - TIFF format (no metadata)

+

Responses

Request samples

Content type
application/json
{
  • "overwrite": false,
  • "format": "NXmxOnlyData"
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get file writer settings

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "overwrite": false,
  • "format": "NXmxOnlyData"
}

Change instrument metadata

This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
source_name
required
string
source_type
string
Default: ""

Type of radiation source. NXmx gives a fixed dictionary, though Jungfraujoch is not enforcing compliance. +https://manual.nexusformat.org/classes/base_classes/NXsource.html#nxsource +NXsource allows the following:

+

Spallation Neutron Source +Pulsed Reactor Neutron Source +Reactor Neutron Source +Synchrotron X-ray Source +Pulsed Muon Source +Rotating Anode X-ray +Fixed Tube X-ray +UV Laser +Free-Electron Laser +Optical Laser +Ion Source +UV Plasma Source +Metal Jet X-ray

+
instrument_name
required
string
pulsed_source
boolean
Default: false

Settings specific to XFEL (e.g., every image has to come from TTL trigger, save pulse ID and event code)

+
electron_source
boolean
Default: false

Settings specific to electron source (e.g., wavelength definition)

+

Responses

Request samples

Content type
application/json
{
  • "source_name": "Swiss Light Source",
  • "source_type": "Synchrotron X-ray Source",
  • "instrument_name": "CristallinaMX",
  • "pulsed_source": false,
  • "electron_source": false
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get instrument metadata

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "source_name": "Swiss Light Source",
  • "source_type": "Synchrotron X-ray Source",
  • "instrument_name": "CristallinaMX",
  • "pulsed_source": false,
  • "electron_source": false
}

Change image output format

This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
summation
required
boolean

Enable summation of images to a given image_time +If disabled images are saved according to original detector speed, but image count is adjusted

+
geometry_transform
required
boolean

Place module read-out into their location on composed detector and extend multipixels

+
jungfrau_conversion
required
boolean

Convert pixel value in ADU to photon counts/energy +Only affects JUNGFRAU detector

+
jungfrau_conversion_factor_keV
number <float> [ 0.001 .. 500 ]

Used to convert energy deposited into pixel to counts +If not provided incident_energy_keV is used

+
bit_depth_image
integer <int64>
Enum: 8 16 32

Bit depth of resulting image (it doesn't affect the detector read-out value) +If not provided value is adjusted automatically

+
signed_output
boolean

Controls if pixels have signed output +If not provided value is adjusted automatically

+
mask_module_edges
required
boolean
Default: true

Mask 1 pixel on the module boundary

+
mask_chip_edges
required
boolean
Default: true

Mask multipixels on chip boundary

+
jungfrau_mask_pixels_without_g0
boolean
Default: true

JUNGFRAU: mask pixels that don't operate in G0, but do operate in G1 and G1. +This should be turned off for cases, where detector is operated at room temperature with long exposure time.

+
apply_mask
required
boolean
Default: false

Masked pixels are set to special value in the images produced by Jungfraujoch

+
jungfrau_pedestal_g0_rms_limit
integer <int64> >= 0
Default: 100

Pixels with pedestal G0 RMS above the threshold are marked as masked pixels

+

Responses

Request samples

Content type
application/json
{
  • "summation": true,
  • "geometry_transform": true,
  • "jungfrau_conversion": true,
  • "jungfrau_conversion_factor_keV": 0.001,
  • "bit_depth_image": 8,
  • "signed_output": true,
  • "mask_module_edges": true,
  • "mask_chip_edges": true,
  • "jungfrau_mask_pixels_without_g0": true,
  • "apply_mask": false,
  • "jungfrau_pedestal_g0_rms_limit": 100
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get image output format

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "summation": true,
  • "geometry_transform": true,
  • "jungfrau_conversion": true,
  • "jungfrau_conversion_factor_keV": 0.001,
  • "bit_depth_image": 8,
  • "signed_output": true,
  • "mask_module_edges": true,
  • "mask_chip_edges": true,
  • "jungfrau_mask_pixels_without_g0": true,
  • "apply_mask": false,
  • "jungfrau_pedestal_g0_rms_limit": 100
}

Configure format for raw data collection

This can only be done when detector is Idle, Error or Inactive states.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Configure format for data collection with full conversion

This can only be done when detector is Idle, Error or Inactive states.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Configure spot finding

Can be done anytime, also while data collection is running

+
Request Body schema: application/json
enable
required
boolean
Default: true

Enable spot finding. This is temporary setting, i.e. can be changed anytime during data collection. +Even if disabled spot finding information will still be send and written, though always with zero spots.

+
indexing
required
boolean
Default: true

Enable indexing. This is temporary setting, i.e. can be changed anytime during data collection.

+
signal_to_noise_threshold
required
number <float> >= 0
photon_count_threshold
required
integer <int64> >= 0
min_pix_per_spot
required
integer <int64> >= 1
max_pix_per_spot
required
integer <int64> >= 1
high_resolution_limit
required
number <float>

High resolution limit for spot finding [Angstrom]

+
low_resolution_limit
required
number <float>

Low resolution limit for spot finding [Angstrom]

+
high_resolution_limit_for_spot_count_low_res
required
number <float> [ 2 .. 8 ]

High resolution threshold to consider spot "low resolution" [Angstrom]

+
quick_integration
required
boolean
Default: false

Quick integration of Bragg spots in diffraction images. +If enabled it will likely reduce performance of Jungfraujoch for datasets with a very high indexing rate. +(experimental feature)

+
ice_ring_width_q_recipA
required
number <float> [ 0 .. 1 ]
Default: 0.02

Width of ice ring in q-space in reciprocal space

+
high_res_gap_Q_recipA
number <float> [ 0.1 .. 5 ]
Default: 1.5

This parameter is used to remove spurious spots at a very high resolution, that sometimes appear due to very low background close to the edge of the detector. +If there is a gap in (1/d)-space between spots of at least this size, spots on the side of the gap with high resolution will be discarded. This is optional parameter. +This option should be turned OFF for small molecule datasets or for crystals with very low mosaicity, when it is expected to see only few spots in any case.

+

Responses

Request samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5
}

Get data processing configuration

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5
}

Configure azimuthal integration

Can be done when detector is Inactive or Idle

+
Request Body schema: application/json
polarization_corr
required
boolean
Default: true

Apply polarization correction for azimuthal integration (polarization factor must be configured in dataset settings)

+
solid_angle_corr
required
boolean
Default: true

Apply solid angle correction for azimuthal integration

+
high_q_recipA
required
number <float> [ 0.00002 .. 10 ]
low_q_recipA
required
number <float> [ 0.00001 .. 10 ]
q_spacing
required
number <float> >= 0.00001
azimuthal_bins
integer <int64> [ 1 .. 512 ]
Default: 1

Numer of azimuthal (phi) bins; 1 = standard 1D azimuthal integration

+
force_cpu
boolean
Default: false

Force CPU processing of azimuthal integration in the FPGA data acquisition workflow. +This allows to extend number of azimuthal integration bins, as well as to calculate standard deviation +of the azimuthal integration results.

+

Responses

Request samples

Content type
application/json
{
  • "polarization_corr": true,
  • "solid_angle_corr": true,
  • "high_q_recipA": 0.00002,
  • "low_q_recipA": 0.00001,
  • "q_spacing": 0.00001,
  • "azimuthal_bins": 1,
  • "force_cpu": false
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get azimuthal integration configuration

Can be done anytime

+

Responses

Response samples

Content type
application/json
{
  • "polarization_corr": true,
  • "solid_angle_corr": true,
  • "high_q_recipA": 0.00002,
  • "low_q_recipA": 0.00001,
  • "q_spacing": 0.00001,
  • "azimuthal_bins": 1,
  • "force_cpu": false
}

Load binary image for internal FPGA generator

Load image for internal FPGA generator. This can only happen in Idle state of the detector. +Requires binary blob with 16-bit integer numbers of size of detector in raw/converted coordinates +(depending on detector settings).

+
query Parameters
id
integer <int64> [ 0 .. 127 ]

Image id to upload

+
Request Body schema: application/octet-stream
string <binary>

Responses

Load TIFF image for internal FPGA generator

Load image for internal FPGA generator. This can only happen in Idle state of the detector. +Requires TIFF with 16-bit integer numbers of size of detector in raw/converted coordinates +(depending on detector settings).

+
query Parameters
id
integer [ 0 .. 127 ]

Image ID to upload

+
Request Body schema: image/tiff
string <binary>

Responses

Select detector

Jungfraujoch allows to control multiple detectors and/or region-of-interests. +The command allows to choose one detector from the list (ID has to be consistent with one provided by GET response). +Changing detector will set detector to Inactive state and will require reinitialization.

+
Request Body schema: application/json
id
required
integer <int64>

Responses

Request samples

Content type
application/json
{
  • "id": 1
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

List available detectors

Configured detectors that can be selected by used

+

Responses

Response samples

Content type
application/json
{
  • "detectors": [
    ],
  • "current_id": 0
}

Set ZeroMQ preview settings

Jungfraujoch can generate preview message stream on ZeroMQ SUB socket. +Here settings of the socket can be adjusted. +While the data structure contains also socket_address, this cannot be changed via HTTP and is ignore in PUT request. +Options set with this PUT request have no effect on HTTP based preview.

+
Request Body schema: application/json
enabled
required
boolean
Default: true

ZeroMQ preview socket is enabled.

+
period_ms
required
integer <int64>
Default: 1000

Period for generating preview image sent to the ZeroMQ interface in milliseconds. Default is 1 second. +If set to zero, all images will be sent ZeroMQ (should be used only in case of relatively slow data collection). +This has no effect on HTTP based preview, which updates always at rate of 1 second.

+
socket_address
string

PUB ZeroMQ socket for preview images. This socket operates at a reduced frame rate. +Images are serialized using CBOR. +Address follows ZeroMQ convention for sockets - in practice ipc:// and tcp://: sockets are OK. +0.0.0.0 instead of IP address is accepted and means listening on all network interfaces.

+

Responses

Request samples

Content type
application/json
{
  • "enabled": true,
  • "period_ms": 1000,
  • "socket_address": "string"
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get ZeroMQ preview settings

Responses

Response samples

Content type
application/json
{
  • "enabled": true,
  • "period_ms": 1000,
  • "socket_address": "string"
}

Set ZeroMQ metadata settings

Jungfraujoch can generate metadata message stream on ZeroMQ PUB socket. This stream covers all images. +Here settings of the socket can be adjusted. +While the data structure contains also socket_address, this cannot be changed via HTTP and is ignore in PUT request.

+
Request Body schema: application/json
enabled
required
boolean
Default: true

ZeroMQ metadata socket is enabled.

+
period_ms
required
integer <int64> >= 1
Default: 1000

Period for generating metadata package sent to the ZeroMQ interface in milliseconds.

+
socket_address
string

PUB ZeroMQ socket for image metadata information. +Image metadata are serialized using CBOR. +Address follows ZeroMQ convention for sockets - in practice ipc:// and tcp://: sockets are OK. +0.0.0.0 instead of IP address is accepted and means listening on all network interfaces.

+

Responses

Request samples

Content type
application/json
{
  • "enabled": true,
  • "period_ms": 1000,
  • "socket_address": "string"
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get ZeroMQ metadata socket settings

Responses

Response samples

Content type
application/json
{
  • "enabled": true,
  • "period_ms": 1000,
  • "socket_address": "string"
}

Set configuration for dark data collection to calculate mask

This is only possible when operating DECTRIS detectors at the moment; it will be also available for PSI EIGER at some point. +This can only be done when detector is Idle, Error or Inactive states.

+
Request Body schema: application/json
detector_threshold_keV
required
number <float> [ 2.5 .. 100 ]
Default: 3.5

Energy threshold for dark image collection

+
frame_time_us
required
integer <int64> [ 500 .. 100000 ]
Default: 10000

Time between frames for dark image collection

+
number_of_frames
required
integer <int64> >= 0
Default: 1000

Number of frames for dark image collection; zero means no dark collection

+
max_allowed_pixel_count
required
integer <int64> >= 0
Default: 1

Maximum count in a pixel considered normal (not-masked)

+
max_frames_with_signal
required
integer <int64> >= 0
Default: 10

Maximum number of frames with signal in a pixel considered normal (not-masked)

+

Responses

Request samples

Content type
application/json
{
  • "detector_threshold_keV": 3.5,
  • "frame_time_us": 10000,
  • "number_of_frames": 1000,
  • "max_allowed_pixel_count": 1,
  • "max_frames_with_signal": 10
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get settings for dark data collection to calculate mask

Responses

Response samples

Content type
application/json
{
  • "detector_threshold_keV": 3.5,
  • "frame_time_us": 10000,
  • "number_of_frames": 1000,
  • "max_allowed_pixel_count": 1,
  • "max_frames_with_signal": 10
}

Get Jungfraujoch status

Status of the data acquisition

+

Responses

Response samples

Content type
application/json
{
  • "state": "Inactive",
  • "progress": 1,
  • "message": "string",
  • "message_severity": "success",
  • "gpu_count": 0,
  • "broker_version": "1.0.0-rc.128"
}

Get status of FPGA devices

Responses

Response samples

Content type
application/json
[
  • {
    }
]

Return XFEL pulse IDs for the current data acquisition

Return array of XFEL pulse IDs - (-1) if image not recorded

+

Responses

Response samples

Content type
application/json
[
  • 0
]

Return XFEL event codes for the current data acquisition

Return array of XFEL event codes

+

Responses

Response samples

Content type
application/json
[
  • 0
]

Get status of image pusher

Responses

Response samples

Content type
application/json
{
  • "pusher_type": "ZeroMQ",
  • "addr": [
    ],
  • "connected_writers": 0,
  • "images_written": 0,
  • "images_write_error": 0,
  • "writer_fifo_utilization": [
    ]
}

Get detector status

Status of the JUNGFRAU detector

+

Responses

Response samples

Content type
application/json
{
  • "state": "Idle",
  • "powerchip": "PowerOn",
  • "server_version": "string",
  • "number_of_triggers_left": 0,
  • "fpga_temp_degC": [
    ],
  • "high_voltage_V": [
    ]
}

Get ROI definitions

Responses

Response samples

Content type
application/json
{
  • "box": {
    },
  • "circle": {
    },
  • "azim": {
    }
}

Upload ROI definitions

Request Body schema: application/json
required
object (roi_box_list)

List of box ROIs

+
required
object (roi_circle_list)

List of circular ROIs

+
required
object (roi_azim_list)

List of azimuthal ROIs

+

Responses

Request samples

Content type
application/json
{
  • "box": {
    },
  • "circle": {
    },
  • "azim": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get general statistics

Responses

Response samples

Content type
application/json
{
  • "detector": {
    },
  • "detector_list": {
    },
  • "detector_settings": {
    },
  • "image_format_settings": {
    },
  • "instrument_metadata": {
    },
  • "file_writer_settings": {
    },
  • "data_processing_settings": {
    },
  • "measurement": {
    },
  • "broker": {
    },
  • "fpga": [
    ],
  • "calibration": [
    ],
  • "zeromq_preview": {
    },
  • "zeromq_metadata": {
    },
  • "dark_mask": {
    },
  • "pixel_mask": {
    },
  • "roi": {
    },
  • "az_int": {
    },
  • "buffer": {
    },
  • "indexing": {
    },
  • "bragg_integration": {
    },
  • "image_pusher": {
    }
}

Get data collection statistics

Results of the last data collection

+

Responses

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "run_number": 0,
  • "experiment_group": "string",
  • "images_expected": 0,
  • "images_collected": 0,
  • "images_sent": 0,
  • "images_written": 0,
  • "images_discarded_lossy_compression": 0,
  • "max_image_number_sent": 0,
  • "collection_efficiency": 1,
  • "compression_ratio": 5.3,
  • "cancelled": true,
  • "max_receiver_delay": 0,
  • "indexing_rate": 0.1,
  • "detector_width": 0,
  • "detector_height": 0,
  • "detector_pixel_depth": 2,
  • "bkg_estimate": 0.1,
  • "unit_cell": "string",
  • "error_pixels": 0.1,
  • "saturated_pixels": 0.1,
  • "roi_beam_pixels": 0.1,
  • "roi_beam_sum": 0.1
}

Get calibration statistics

Statistics are provided for each module/storage cell separately

+

Responses

Response samples

Content type
application/json
[
  • {
    }
]

Get mask of the detector (binary)

Detector must be Initialized. +Get full pixel mask of the detector. +See NXmx standard for meaning of pixel values.

+

Responses

Detector must be Initialized. +Get user mask of the detector (binary) +

Get user pixel mask of the detector in the actual detector coordinates: 0 - good pixel, 1 - masked

+

Responses

Upload user mask of the detector (binary)

Should be in Idle state. +Upload user mask of the detector - this is for example to account for beam stop shadow or misbehaving regions. +If detector is conversion mode the mask can be both in raw (1024x512; stacked modules) or converted coordinates. +In the latter case - module gaps are ignored and don't need to be assigned value. +Mask is expected as binary array (4-byte; unsigned). +0 - good pixel, other value - masked +User mask is stored in NXmx pixel mask (bit 8), as well as used in spot finding and azimuthal integration.

+
Request Body schema: application/octet-stream
string <binary>

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get mask of the detector (TIFF)

Should be in Idle state. +Get full pixel mask of the detector +See NXmx standard for meaning of pixel values

+

Responses

Detector must be Initialized. +Get user mask of the detector (TIFF) +

Get user pixel mask of the detector in the actual detector coordinates: 0 - good pixel, 1 - masked

+

Responses

Upload user mask of the detector

Should be in Idle state. +Upload user mask of the detector - this is for example to account for beam stop shadow or misbehaving regions. +If detector is conversion mode the mask can be both in raw (1024x512; stacked modules) or converted coordinates. +In the latter case - module gaps are ignored and don't need to be assigned value. +Mask is expected as a single-channel TIFF (8-, 16- or 32-bit integer, signed or unsigned). +0 - good pixel, other value - masked +User mask is stored in NXmx pixel mask (bit 8), as well as used in spot finding and azimuthal integration. +User mask is not automatically applied - i.e. pixels with user mask will have a valid pixel value in the images.

+
Request Body schema: image/tiff
string <binary>

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get pedestal in TIFF format

query Parameters
gain_level
required
integer

Gain level (0, 1, 2)

+
sc
integer

Storage cell number

+

Responses

Generate 1D plot from Jungfraujoch

query Parameters
binning
integer
Default: 1

Binning of frames for the plot (0 = default binning)

+
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b" "compression_ratio" "ice_ring_score"

Type of requested plot

+
fill
number <float>

Fill value for elements that were missed during data collection

+
experimental_coord
boolean
Default: false

If measurement has goniometer axis defined, plot X-axis will represent rotation angle +If measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value +For still measurement the number is ignored

+
azint_unit
string
Default: "Q_recipA"
Enum: "Q_recipA" "d_A" "two_theta_deg"

Unit used for azim int.

+

Responses

Response samples

Content type
application/json
{
  • "title": "string",
  • "unit_x": "image_number",
  • "size_x": 0.1,
  • "size_y": 0.1,
  • "plot": [
    ]
}

Generate 1D plot from Jungfraujoch and send in raw binary format. +Data are provided as (32-bit) float binary array. +This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. +

query Parameters
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b" "compression_ratio" "ice_ring_score"

Type of requested plot

+
roi
string non-empty

Name of ROI for which plot is requested

+

Responses

Get full scan result

Responses

Response samples

Content type
application/json
{
  • "file_prefix": "string",
  • "rotation_unit_cell": {
    },
  • "rotation_crystal_lattice": [
    ],
  • "rotation_bravais": "string",
  • "images": [
    ]
}

Get Start message in CBOR format

Contains metadata for a dataset (e.g., experimental geometry)

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get image message in CBOR format

Contains full image data and metadata. The image must come from the latest data collection.

+
query Parameters
id
integer <int64> >= -2
Default: -1

Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get preview image in JPEG format using custom settings

query Parameters
id
integer <int64> >= -2
Default: -1

Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer

+
show_user_mask
boolean
Default: false

Show user mask

+
show_roi
boolean
Default: false

Show ROI areas on the image

+
show_spots
boolean
Default: true

Show spot finding results on the image

+
show_predictions
boolean
Default: false

Show Bragg spot predictions on the image (if available for the image)

+
show_beam_center
boolean
Default: true

Show beam center on the image

+
saturation
number <float> [ -32767 .. 32767 ]

Saturation value to set contrast in the preview image; if not provided, then autocontrast procedure is used

+
jpeg_quality
integer <int64> [ 0 .. 100 ]
Default: 100

Quality of JPEG image (100 - highest; 0 - lowest)

+
show_res_ring
number <float> [ 0.1 .. 100 ]
Default: 0.1

Show resolution ring, provided in Angstrom

+
color
string
Default: "indigo"
Enum: "indigo" "viridis" "bw" "wb" "green" "heat" "magma" "inferno"

Color scale for preview image

+
show_res_est
boolean
Default: false

Show resolution estimation as a ring

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get preview image in TIFF format

query Parameters
id
integer <int64> >= -2
Default: -1

Image ID in the image buffer. Special values: -1 - last image in the buffer, -2: last indexed image in the buffer

+

Responses

Clear image buffer

Turns off image buffer for the last data collection. Can be only run when Jungfraujoch is not collecting data.

+

Responses

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get status of the image buffers

Can be run at any stage of Jungfraujoch operation, including during data collection. +The status of the image buffer is volatile during data collection - if data collection goes for more images than available buffer slots, +then image might be replaced in the buffer between calling /images and /image.cbor.

+

Responses

Response samples

Content type
application/json
{
  • "min_image_number": 0,
  • "max_image_number": 0,
  • "image_numbers": [
    ],
  • "total_slots": 0,
  • "available_slots": 0,
  • "in_preparation_slots": 0,
  • "in_sending_slots": 0,
  • "current_counter": 0
}

Get Jungfraujoch version of jfjoch_broker

Responses

+ + + + diff --git a/_images/jfjoch.png b/_images/jfjoch.png new file mode 100644 index 00000000..aa75a7ea Binary files /dev/null and b/_images/jfjoch.png differ diff --git a/_sources/ACKNOWLEDGEMENT.md.txt b/_sources/ACKNOWLEDGEMENT.md.txt new file mode 100644 index 00000000..56807b32 --- /dev/null +++ b/_sources/ACKNOWLEDGEMENT.md.txt @@ -0,0 +1,10 @@ +# Acknowledgements + +Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). + +The project is supported by : +* Innosuisse via Innovation Project "NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications" (101.535.1 IP-ENG; Apr 2023 - Sep 2025). +* ETH Domain via Open Research Data Contribute project (Jan - Dec 2023) +* AMD University Program with donation of licenses of Ethernet IP cores and Vivado software + +This software uses Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license diff --git a/_sources/CBOR.md.txt b/_sources/CBOR.md.txt new file mode 100644 index 00000000..5f3fd2c5 --- /dev/null +++ b/_sources/CBOR.md.txt @@ -0,0 +1,366 @@ +# CBOR messages + +To communicate between FPGA-equipped receiver system and writers, +Jungfraujoch is using binary CBOR encoding with tinycbor library (Intel). +The protocol is based on and compatible with [DECTRIS Stream2](https://github.com/dectris/documentation/tree/main/stream_v2). +There are minor differences at the moment: + +* LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed +* Few fields are currently absent +* Extra fields are present beyond DECTRIS standard +* There are calibration and metadata messages defined beyond DECTRIS specification + +## Start message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "start" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| detector_distance | float | Detector distance \[m\] | | +| detector_translation | Array(float) | Detector translation vector \[m\] | X | +| beam_center_x | float | Beam center in X direction \[pixels\] | X | +| beam_center_y | float | Beam center in Y direction \[pixels\] | X | +| countrate_correction_enabled | bool | Countrate correction enabled | X | +| flatfield_enabled | bool | Flatfield enabled | X | +| number_of_images | uint64 | Number of images in the series | X | +| image_size_x | uint64 | Image width \[pixels\] | X | +| image_size_y | uint64 | Image height \[pixels\] | X | +| incident_energy | float | X-ray energy \[eV\] | X | +| incident_wavelength | float | X-ray wavelength \[Angstrom\] | X | +| incident_wavelength_spread | float (optional) | FWHM of the X-ray wavelength distribution \[Angstrom\] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic | | +| frame_time | float | Frame time, if multiple frames per trigger \[s\] | X | +| count_time | float | Exposure time \[s\] | X | +| saturation_value | int64 | Maximum valid sample value | X | +| error_value | int64 (optional) | Value used in images to describe pixels that are in error state or missing | | +| pixel_size_x | float | Pixel width \[m\] | X | +| pixel_size_y | float | Pixel height \[m\] | X | +| sensor_thickness | float | Sensor thickness \[m\] | X | +| sensor_material | string | Sensor material | X | +| arm_date | date | Approximate date of arming | X | +| pixel_mask_enabled | bool | Pixel mask applied on images | X | +| detector_description | string | Name of the detector | X | +| detector_serial_number | string | Detector serial number | X | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| fluorescence | object (optional) | X-ray fluorescence spectrum collected at start | | +| - energy | Array(float) | Energy of measuring point \[eV\] | | +| - data | Array(float) | Fluorescence scan result `data` \[arbitrary units\]; must be strictly the same length as energy | | +| goniometer | Map | Definition of rotation axis (optional) | X | +| - `AXIS` | string | Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch | X | +| - - increment | float | Rotation axis increment (per image) in degree \[deg\] | X | +| - - start | float | Rotation axis start angle \[deg\] | X | +| - - axis | Array(float) | Vector for the rotation axis | | +| - - helical_step | Array(float) | Translation for helical scan for 1 image \[m\] | | +| - - screening_wedge | Array(float) | Wedge for screening \[deg\] (increment would correspond to difference between screening points) | | +| grid_scan | object | Grid scan definition (optional and exclusive with rotation axis) | | +| - n_fast | uint64 | Number of elements along fast axis | | +| - n_slow | uint64 | Number of elements along slow axis | | +| - step_x_axis | float | Step along X axis, can be negative \[m\] | | +| - step_y_axis | float | Step along Y axis, can be negative \[m\] | | +| - snake_scan | bool | Snake scan (rows alternate direction) | | +| - vertical_scan | bool | Vertical scan (enabled: fast direction = Y, disabled: fast direction = X) | | +| jungfrau_conversion_enabled | bool (optional) | Applying JUNGFRAU pixel conversion (to photons or keV) | | +| jungfrau_conversion_factor | float (optional) | Factor used for JUNGFRAU conversion \[eV\] | | +| geometry_transformation_enabled | bool (optional) | Transformation from detector module geometry (512x1024) to full detector geometry | | +| pixel_mask | Map(string -> Image) | Pixel mask - multiple in case of storage cells | X | +| channels | Array(string) | List of image channels | X | +| max_spot_count | uint64 | Maximum number of spots identified in spot finding | | +| max_extra_lattices | uint64 | Maximum number of extra lattices | | +| storage_cell_number | uint64 (optional) | Number of storage cells used by JUNGFRAU | | +| storage_cell_delay | Rational | Delay of storage cells in JUNGFRAU | | +| threshold_energy | Map(string -> float) | Per-channel threshold energy \[eV\] (map of channel name to value) | | +| image_dtype | string | Pixel bit type (e.g. uint16) | X | +| unit_cell | object (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| az_int_q_bin_count | uint64 | Number of azimuthal integration bins in the radial direction | | +| az_int_phi_bin_count | uint64 | Number of azimuthal integration bins in the phi angle direction | | +| az_int_bin_to_q | Array(float) | Q value for each azimuthal integration bin \[angstrom^-1\] | | +| az_int_bin_to_two_theta | Array(float) | Two theta angle value for each azimuthal integration bin \[deg\] | | +| az_int_bin_to_phi | Array(float) | Phi value for each azimuthal integration bin \[deg\] | | +| az_int_map | Image | Mapping between pixel and bin number | | +| summation | uint64 | Factor of frame summation | | +| user_data | string | JSON serialized to string that can contain the following fields (all fields are optional): | X | +| - file_prefix | string | File prefix | | +| - images_per_file | uint64 | Number of images written per file | | +| - images_per_trigger | uint64 | Number of images collected per trigger | | +| - source_name | string | Facility name | | +| - source_type | string | Type of X-ray source (use NXsource/type values, for example "Synchrotron X-ray Source" or "Free-Electron Laser") | | +| - instrument_name | string | Instrument name | | +| - sample_name | string | Name of the sample | | +| - user | any valid JSON | Value of header_appendix provided at collection start to Jungfraujoch | | +| - attenuator_transmission | float | Attenuator transmission \[\] | | +| - total_flux | float | Total flux \[ph/s\] | | +| - space_group_number | uint64 | Space group number | | +| - summation_mode | string | Summation mode (internal\|fpga\|cpu) | | +| - overwrite | bool | Overwrite existing HDF5 files | | +| - file_format | int | File writer format: 0 = only data files, 1 = NXmx legacy soft links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF, 5 = TIFF, 6 = no file written | | +| - roi | Array(object) | ROI configurations; each element is one of: | | +| | | type "box": xmin, xmax, ymin, ymax (numbers) | | +| | | type "circle": r, x, y (numbers) | | +| | | type "azim": qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector | | +| - gain_file_names | Array(string) | Names of JUNGFRAU gain files used for the current detector | | +| - write_master_file | bool | With multiple sockets, it selects which socket will provide master file | | +| - write_images | bool | Write images in the HDF5 file (if false, will only write metadata) | | +| - data_reduction_factor_serialmx | uint64 | Data reduction factor for serial MX | | +| - experiment_group | string | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number | | +| - jfjoch_release | string | Jungfraujoch release number | | +| - socket_number | uint64 | Number of ZeroMQ socket (on `jfjoch_broker` side) used for transmission | | +| - bit_depth_readout | uint64 | Bit depth of the detector readout | | +| - writer_notification_zmq_addr | string | ZeroMQ address to inform `jfjoch_broker` about writers that finished operation | | +| - xfel_pulse_id | uint64 | Pulse IDs are recorded for images | | +| - ring_current_mA | float | Ring current at the start of the measurement | | +| - sample_temperature_K | float | Sample temperature \[K\] | | +| - detect_ice_rings | bool | Ice ring detection feature is enabled | | +| - indexing_algorithm | string | Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none | | +| - geom_refinement_algorithm | string | Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center | | +| - poni_rot1 | float | Tilt of the detector rot1 according to PyFAI PONI convention \[rad\] | | +| - poni_rot2 | float | Tilt of the detector rot2 according to PyFAI PONI convention \[rad\] | | +| - poni_rot3 | float | Tilt of the detector rot3 according to PyFAI PONI convention \[rad\] | | + +See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) for definition of Image as MultiDimArray with optional compression. + +## Image message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|-----------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "image" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| real_time | Rational | Exposure time | X | | +| start_time | Rational | Exposure start time (highly approximate) | X | | +| end_time | Rational | Exposure end time (highly approximate) | X | | +| spots | Array(object) | Spots: | | | +| - x | float | observed position in x (pixels) | | | +| - y | float | observed position in y (pixels) | | | +| - I | float | intensity (photons) | | | +| - maxc | int64 | max count (photons) | | | +| - ice_ring | bool | spot in resolution range for ice rings | | | +| - indexed | bool | indexed solution | | | +| - latt | int64 | Lattice to which the peak belongs (negative number = not indexed) | | | +| - image | int64 | image number the spot belongs to | | | +| - h | int64 | Miller index (indexed spots only) | | | +| - k | int64 | Miller index (indexed spots only) | | | +| - l | int64 | Miller index (indexed spots only) | | | +| - dist_ewald | float | distance to Ewald sphere \[Angstrom^-1\] (indexed spots only) | | | +| reflections | Array(object) | Reflections: | | | +| - h | int64 | Miller index | | | +| - k | int64 | Miller index | | | +| - l | int64 | Miller index | | | +| - x | float | prediced position in x (pixels) | | | +| - y | float | predicted position in y (pixels) | | | +| - obs_x | float | observed position in x (pixels) | | | +| - obs_y | float | observed position in y (pixels) | | | +| - d | float | resolution \[Angstrom\] | | | +| - I | float | integrated intensity (photons) | | | +| - bkg | float | mean background value (photons) | | | +| - sigma | float | standard deviation, estimated from counting statistics (photons) | | | +| - image | float | image number (present for each spot) | | | +| - rp | float | Distance to Ewald sphere \[Angstrom^-1\] | | | +| - rlp | float | Reciprocal Lorentz and polarization corrections | | | +| - partiality | float | Partiality of the reflection | | | +| - phi | float | phi angle from XDS: difference from middle of current frame, not absolute \[deg\] | | | +| - zeta | float | Lorentz zeta factor (reciprocal-space geometry term) | | | +| - image_scale_corr | float | Per-image scale correction; I_true = image_scale_corr * I | | | +| spot_count | uint64 | Spot count | | | +| spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | +| spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | +| az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| | | NaN is used for empty bins and has to be taken care by the receiver | | | +| az_int_profile_std | Array(float) | Standard deviation for azimuthal integration. (NaN for less than 2 samples) | | | +| az_int_profile_count | Array(uint64) | Number of pixels contributing to azimuthal bin | | | +| indexing_result | bool | Indexing successful | | | +| indexing_lattice_count | int64 | Number of indexing lattices found for this image | | | +| indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| indexing_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | | +| indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| profile_radius | float | Profile radius of the image - describes distance of observed reflections from the Ewald sphere \[Angstrom^-1\] | | | +| integrated_reflections | int64 | Count of integrated reflections | | | +| mosaicity | float | Angular range of spots in image from a rotation scan \[degree\] | | | +| b_factor | float | Estimated B-factor (Angstrom^2) | | | +| compression_time | float | Time spent on compression/decompressing image \[s\] | | | +| preprocessing_time | float | Time spent on preparing the image for analysis \[s\] | | | +| azint_time | float | Time spent on azimuthal integration \[s\] | | | +| spot_finding_time | float | Time spent on spot finding \[s\] | | | +| indexing_time | float | Time spent on indexing \[s\] | | | +| refinement_time | float | Time spent on refinement of indexing solution and experimental geometry \[s\] | | | +| index_analysis_time | float | Time spent on analyzing idnexing solution, calculating profile radius and mosaicity \[s\] | | | +| bragg_prediction_time | float | Time spent on predicting Bragg spots \[s\] | | | +| integration_time | float | Time spent on Bragg integration \[s\] | | | +| image_scale_time | float | Time spent on on-the-fly scaling \[s\] | | | +| processing_time | float | Total processing time \[s\] | | | +| xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| lattice_type | object | Bravais lattice classification of the indexing result (present only if available) | | X | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | +| jf_info | uint64 | Detector info field | | | +| receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| receiver_free_send_buf | uint64 | Receiver internal number of available buffer locations | | | +| receiver_buf_in_sending | uint64 | Receiver internal number of buffer locations currently in sending/writing | | | +| receiver_buf_in_preparation | uint64 | Receiver internal number of buffer locations currently in processing | | | +| storage_cell | uint64 | Storage cell number | | | +| saturated_pixel_count | uint64 | Saturated pixel count | | | +| pixel_sum | uint64 | Sum of all pixels, excl. error and saturation | | | +| error_pixel_count | uint64 | Error pixel count | | | +| strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| min_viable_pixel_value | int64 | Minimal pixel value, excl. error and saturation | | | +| max_viable_pixel_value | int64 | Maximal pixel value, excl. error and saturation | | | +| resolution_estimate | float | Diffraction resolution estimation \[Angstrom\] | | X | +| data_collection_efficiency | float | Image collection efficiency \[\] | | | +| packets_expected | uint64 | Number of packets expected per image (in units of 2 kB) | | | +| packets_received | uint64 | Number of packets received per image (in units of 2 kB) | | | +| bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] | | | +| ice_ring_score | float | Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | | | +| beam_corr_x | float | Beam center correction X applied during processing \[pixel\] | | X | +| beam_corr_y | float | Beam center correction Y applied during processing \[pixel\] | | X | +| image_scale_factor | float | Scaling result: Image scale factor (g) | | X | +| image_scale_mosaicity | float | Scaling result: Image scale mosaicity \[deg\] | | X | +| image_scale_b_factor | float | Scaling result: Image scale B factor \[Angstrom^2\] | | X | +| image_scale_cc | float | Scaling result: Image scale CC | | X | +| adu_histogram | Array(uint64) | ADU histogram | | | +| roi_integrals | object | Results of ROI calculation | | X | +| - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - pixels | uint64 | Valid pixels in ROI area | | | +| - max_count | int64 | Highest count in ROI area \[photons\] | | | +| - x_weighted_sum | int64 | ROI pixel X position multiplied by photon count \[photons * pixels\] | | | +| - y_weighted_sum | int64 | ROI pixel Y position multiplied by photon count \[photons * pixels\] | | | +| user_data | string | Optional user defined text information - this is image_appendix serialized to JSON format | X | | +| data | Map(string -> Image) | Image | X | | + +## Metadata message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|------------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "metadata" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| images | Array(object) | Array of images (order and size of the array are not guaranteed) | X | | +| - image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| - original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| - real_time | Rational | Exposure time | X | | +| - start_time | Rational | Exposure start time (highly approximate) | X | | +| - end_time | Rational | Exposure end time (highly approximate) | X | | +| - spot_count | uint64 | Spot count | | | +| - spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| - az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| - indexing_result | bool | Indexing successful | | | +| - indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| - indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| - xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| - xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| - jf_info | uint64 | Detector info field | | | +| - receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| - receiver_free_send_buf | uint64 | Receiver internal number of available send buffers | | | +| - storage_cell | uint64 | Storage cell number | | | +| - saturated_pixel_count | uint64 | Saturated pixel count | | | +| - error_pixel_count | uint64 | Error pixel count | | | +| - strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| - data_collection_efficiency | float | Image collection efficiency \[\] | | | +| - bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] (with solid angle/polarization corrections, if applied) | | X | +| - resolution_estimate | float | Diffraction resolution estimation | | X | +| - adu_histogram | Array(uint64) | ADU histogram | | X | +| - roi_integrals | object | Results of ROI calculation | | X | +| - - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - - pixels | uint64 | Valid pixels in ROI area | | | +| - - max_count | int64 | Highest count in ROI area \[photons\] | | | + +## End message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "end" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| end_date | string | Approximate end date | | +| max_image_number | uint64 | Number of image with the highest number; counted from 1 to distinguish zero images and one image | | +| images_collected | uint64 | Number of images collected | | +| images_sent_to_write | uint64 | Number of images sent to writer; if writer queues were full, it is possible this is less than images collected | | +| data_collection_efficiency | float | Overall network packets collected / network packets expected | | +| az_int_result | Map(text->Array(float)) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | +| adu_histogram | Map(text->Array(uint64)) | ADU values histogram | | +| adu_histogram_bin_width | uint64 | Width of bins in the above histogram \[ADU\] | | +| max_receiver_delay | uint64 | Internal performance of Jungfraujoch | | +| bkg_estimate | float | Mean background estimate for the whole run | | +| indexing_rate | float | Mean indexing rate for the whole run | | +| unit_cell | object (optional) | Unit cell of the system, based on the actual experiment: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| rotation_lattice_type | object | Bravais lattice classification of the total rotation solution over the run, if available; same schema as `lattice_type` | | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | +| rotation_lattice | Array(9 * float) | Real-space lattice basis, flattened 3x3 in row-major order | | +| rotation_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | +| data_collection_efficiency_image | Array(float) | Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context | | +| spot_count | Array(int32) | Per-image spot count | | +| spot_count_ice_ring | Array(int32) | Per-image number of spots within identified ice-ring resolution ranges | | +| spot_count_low_res | Array(int32) | Per-image number of low-resolution spots | | +| spot_count_indexed | Array(int32) | Per-image number of spots fitting indexing solution | | +| image_indexed | Array(uint8) | Per-image indexing result; 0 = not indexed, nonzero = indexed | | +| v_bkg_estimate | Array(float) | Per-image background estimate | | +| ice_ring_score | Array(float) | Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice) | | +| profile_radius | Array(float) | Per-image profile radius \[Angstrom^-1\] | | +| mosaicity | Array(float) | Per-image mosaicity \[degree\] | | +| bFactor | Array(float) | Per-image estimated B-factor \[Angstrom^2\] | | +| resolution_estimate | Array(float) | Per-image diffraction resolution estimate \[Angstrom\] | | +| min_viable_pixel_value | Array(int64) | Per-image minimum valid pixel value, excluding error/saturated pixels | | +| max_viable_pixel_value | Array(int64) | Per-image maximum valid pixel value, excluding error/saturated pixels | | +| saturated_pixel_count | Array(int32) | Per-image saturated pixel count | | +| error_pixel_count | Array(int32) | Per-image error pixel count | | +| image_scale_factor | Array(float) | Per-image scale factor, if scaling/merging was performed | | +| integrated_reflections | Array(int32) | Per-image count of integrated reflections | | +| indexed_lattice_count | Array(int32) | Per-image count of indexed lattices | | +| niggli_class | Array(uint8) | Per-image Niggli class identifier for indexed images; 0 if unavailable | | +| pixel_sum | Array(int64) | Per-image sum of all valid pixels, excluding error/saturated pixels | | +| image_scale_mosaicity | Array(float) | Scaling result: Image scale mosaicity \[deg\] | | +| image_scale_b_factor | Array(float) | Scaling result: Image scale B factor \[Angstrom^2\] | | +| image_scale_cc | Array(float) | Scaling result: Image scale CC | | + +End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted. + +## Calibration message + +| Field name | Type | Description | Present in DECTRIS format | +|--------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "calibration" | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| data | Map(string -> Image) | Calibration map (only single pedestal array per message) | | + +## User data +In many cases there is an interest from facilities to forward more metadata, than available explicitly in the Jungfraujoch. +For this reason two fields can be provided: `header_appendix` (sent with start message) and `image_appendix` (send with image message). +To increase flexibility, both appendices can contain any valid JSON message. +These appendices are serialized into string and stored in CBOR messages as `user_data`. + +Notably for start message, `user_data` can contain more information (non-DECTRIS compliant metadata). +Therefore `user_data` is serialized by Jungfraujoch as CBOR object. There is member `user` which contains `header_appendix` defined in OpenAPI of Jungfraujoch. + +### Notes on images and compression + +- Images are encoded as DECTRIS MultiDimArray with typed array tags: + - For RGB: shape \[3, height, width\], type: u8 + - For grayscale: shape \[height, width\], type according to bit depth and sign (e.g., uint16 LE) +- Compression: + - Uncompressed: raw CBOR byte string + - Bitshuffle+LZ4: tag with \["bslz4", elem_size, bytes\] + - Bitshuffle+Zstandard: tag with \["bszstd", elem_size, bytes\] + +### Notes on typed arrays + +Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays. + +Common tags used in this protocol include: + +- float32 little-endian arrays for `Array(float)` +- uint8 arrays for compact boolean/integer flags such as `image_indexed` +- int32 little-endian arrays for per-image counts +- int64 little-endian arrays for large per-image integer values +- uint64 little-endian arrays for histograms \ No newline at end of file diff --git a/_sources/CHANGELOG.md.txt b/_sources/CHANGELOG.md.txt new file mode 100644 index 00000000..de87e9b6 --- /dev/null +++ b/_sources/CHANGELOG.md.txt @@ -0,0 +1,1081 @@ +# Changelog +## 1.0.0 +### 1.0.0-rc.158 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (`cos^3` of that angle) instead of `cos^3(2*theta)`, so it is correct for a tilted detector and matches PyFAI `solidAngleArray` and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (`XtalOptimizer`) no longer silently ignores an imported PONI `rot3` (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured `rot3` through the full PONI rotation. +* jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting `\\`); write processing outputs next to the input file, with a Browse button and independent `_process.h5` / merged `.mtz`/`.cif` toggles; and show the determined space group in the merge-statistics window. +* jfjoch_viewer: Connect to a broker over `https` (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow. +* jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source. +* rugnux: Accept an absolute `-o` output prefix in offline processing. +* rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged). +* rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing. +* rugnux: De-novo symmetry robustness - don't over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don't flag twinning in holohedral Laue classes where no twin law can exist. +* jfjoch_writer: Fold the refined beam centre into the NXmx detector `translation` vector too (not only the informational `beam_center` fields), so a reprocessed `_process.h5` has a self-consistent refined geometry. +* Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames. +* Packaging: The self-contained Linux viewer `.tgz` now bundles cuFFT, so it runs without a system CUDA toolkit (`.deb`/`.rpm` are unchanged, distro-managed). +* Docs: Documentation updated to match the current analysis code and CLI. + +### 1.0.0-rc.157 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* rugnux: Rebrand the offline data-processing subsystem as `rugnux` and consolidate all offline analysis into the single `rugnux` binary - `jfjoch_process` is now `rugnux`, the former `jfjoch_azint` is now `rugnux --azint-only`, and `jfjoch_scale` is now `rugnux --scale` (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (`--no-merge` disables them), replacing the previous opt-in `-M, --scale-merge`. +* rugnux: CLI fixes - default `-N` to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require `--wavelength > 0`, and correct the reproduced command line and `--scale` reference-cell handling. +* rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics. +* rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection's rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below = 1. +* rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below `--min-captured-fraction` (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned). +* rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved `jfjoch` prefix. +* jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation. +* jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a "reduce frame rate" warning. +* jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint. +* jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed). +* jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types. +* jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails. +* jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak. +* Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation. +* CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*. + +### 1.0.0-rc.156 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output. +* jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed. +* jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections. +* jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw). +* jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path. +* jfjoch_process: Unify still and rotation processing on a single --force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental --reciprocal-profile option is removed. +* jfjoch_process: Add EXPERIMENTAL ice-ring detection (--detect-ice-rings) that excludes ice reflections from scaling. +* jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend. +* jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata. +* jfjoch_writer: Don't write empty grid-scan position arrays when the dataset has no images. +* Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don't fit rather than aborting), and add the jfjoch_recompress tool. +* jfjoch_viewer: Report "Multiple lattices detected" and grey out "Analyze dataset" on a live connection. +* jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons. +* CI: Build Windows (CUDA and non-CUDA) installers. +* CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz. + +### 1.0.0-rc.155 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D) +* jfjoch_viewer: Some graphical improvements. +* jfjoch_viewer: Simplify und unify data analysis settings. +* jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker "dies" in the middle of data acquisition. + +### 1.0.0-rc.154 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability) + +### 1.0.0-rc.153 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing +* jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files +* jfjoch_broker: Add ROI calculation in non-FPGA workflow +* jfjoch_broker: Fixes to TCP image pusher +* jfjoch_broker: Remove NUMA bindings +* jfjoch_broker: Improvements to indexing +* jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values +* jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors +* jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs +* jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results + +### 1.0.0-rc.152 +* jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5) +* jfjoch_viewer: Adjust Q bounds for azimuthal integration +* jfjoch_azint: Add tool to do quick azimuthal integration + +### 1.0.0-rc.151 +* jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname + +### 1.0.0-rc.150 +* jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin. +* jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins. + +### 1.0.0-rc.149 +* XDS plugin: Fix HDF5 mutex to run on multiple processors + +### 1.0.0-rc.148 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Improve azimuthal integration (add calculation) +* jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated) +* jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin + +### 1.0.0-rc.147 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015. +* jfjoch_viewer: Add reciprocal space viewer +* jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset +* jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress) + +### 1.0.0-rc.146 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension) +* jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset. +* jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling. +* jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file. +* jfjoch_scale: Dedicated tool for rescaling/merging existing data. +* jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel. + +WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being. + +### 1.0.0-rc.145 +This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144. + +* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future. +* XDS plugin: Improve performance of VDS reading. +* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD. +* jfjoch_writer: Clean-up of HDF5 routines to better handle issues. + +### 1.0.0-rc.144 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M) + +### 1.0.0-rc.143 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment + +### 1.0.0-rc.142 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8 +* Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent + +### 1.0.0-rc.141 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times +* frontend: Fix selection of FFTW in indexing settings + +### 1.0.0-rc.140 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection +* jfjoch_broker: Minor bug fixes for rare conditions +* jfjoch_process: Significantly improve performance + +### 1.0.0-rc.139 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on `/start` +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on '/start' +* jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite worklow preparation time +* jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received +* jfjoch_broker: Use nanoseconds internally for frame/image/readout time +* jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished) +* jfojch_broker: Improve profiling of different data analysis steps +* jfjoch_broker: Record integration reflection count +* jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds) +* jfjoch_broker: Fix bug where '/wait_till_done' got deadlocked +* jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets + +**Breaking changes**: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time. + +### 1.0.0-rc.138 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time +* jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations +* jfjoch_broker: Goniometer axis name is converted to lowercase +* jfjoch_broker: Fix bug, where wrong HTTP error codes were returned +* jfjoch_process: Improve sigma estimation during merging (K. Takaba) +* jfjoch_process: Modify spot finding thresholds +### 1.0.0-rc.137 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Better track time for each operation in the processing stack +* jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress) +* jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress) +* jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0) + +### 1.0.0-rc.136 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress). + +### 1.0.0-rc.135 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4) +* jfjoch_viewer: Improve image render performance + +### 1.0.0-rc.134 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long +* jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata +* XDS plugin: Add generation of Jungfraujoch plugin for XDS +* CI: Add tests with XDS and DIALS (`xia2.ssx`) + +### 1.0.0-rc.133 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Use httplib for HTTP server instead of Pistache +* jfjoch_broker: Drop OpenSSL support +* jfjoch_broker: Base work for multi-lattice support in the future +* jfjoch_broker: Improve recording time of data analysis steps +* jfjoch_writer: Save per-image information about data analysis timing +* Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib) + +### 1.0.0-rc.132 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* Documentation: Fix equation rendering + +### 1.0.0-rc.131 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS) +* jfjoch_viewer: Fix calibration (pedestal) images being open flipped +* jfjoch_process: Add space group detection (EXPERIMENTAL) + +### 1.0.0-rc.130 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Rotation indexer has two retries if failes +* jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot) +* jfjoch_broker: Integration calculates background mask based on R2 radius +* jfjoch_process: HDF5 files are not saved by default + +### 1.0.0-rc.129 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files) +* jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction +* jfjoch_broker: Image pusher statistics are accessible via the REST interface +* jfjoch_writer: Supports TCP image socket and for these auto-forking option + +### 1.0.0-rc.128 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Handle properly reuse of image buffer locations +* jfjoch_broker: Fix bug in counting idle slots +* jfjoch_broker: Force obtuse angle for monoclinic cells +* jfjoch_process: Change scaling refinement tolerance + +### 1.0.0-rc.127 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Default EIGER readout time is 20 microseconds +* jfjoch_broker: Multiple improvements regarding performance +* jfjoch_broker: Image buffer allows to track frames in preparation and sending +* jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer +* jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets +* jfjoch_writer: Fixes regarding properly closing files in long data collections +* jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code + +### 1.0.0-rc.126 +This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg. + +### 1.0.0-rc.125 +This is an UNSTABLE release. This version adds scalign and merging. These are experimental at the moment, and should not be used for production analysis. +If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Improve logic on switching on/off spot finding +* jfjoch_broker: Increase maximum spot count for FFBIDX to 65536 +* jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD) +* jfjoch_process: Add scalign and merging functionality - program is experimental at the moment and should not be used for production analysis +* jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection +* jfjoch_writer: Save more information about each reflection + +### 1.0.0-rc.124 +This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, +especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123. + +* jfjoch_broker: Improve refection position prediction and Bragg integration code. +* jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation. +* jfjoch_writer: Fix saving mosaicity properly in HDF5 file. +* jfjoch_viewer: Introduce high-dynamic range mode for images +* jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%) +* jfjoch_viewer: Zoom-in numbers have better readability + +### 1.0.0-rc.123 +This is an UNSTABLE release. + +* jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility +* jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M) +* jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M) +* jfjoch_viewer: Add more color schemes for better image readability +* HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable +* HDF5: suppress warning if path (upstream group) doesn't exists when checking if leaf exists + +### 1.0.0-rc.122 +This is an UNSTABLE release. + +* jfjoch_broker: Add thresholding to prefer shorter vectors after FFT +* jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress) +* jfjoch_broker: Update nlohmann::json to 3.12.0 +* jfjoch_viewer: Display file opening errors +* jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available + +### 1.0.0-rc.121 +This is an UNSTABLE release. + +* jfjoch_broker: Report changes in the image buffer, so viewer doesn't reload constantly +* jfjoch_viewer: Improve performance of loading images +* jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes +* jfjoch_viewer: Auto-foreground calculated with histogram +* jfjoch_viewer: Fix rare segmentation fault + +### 1.0.0-rc.120 +This is an UNSTABLE release. + +* jfjoch_broker: Improve performance of binary plot export + +### 1.0.0-rc.119 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Add binary export of data analysis plots over OpenAPI +* jfjoch_broker: Minor fixes to HTTP error handling +* jfjoch_viewer: Prefer binary plots over JSON plots +* jfjoch_viewer: Change foreground with F button + wheel +* jfjoch_viewer: Change way how angles are displayed +* jfjoch_viewer: Display resolution of the mouse cursor in top left corner + +### 1.0.0-rc.118 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement. +* jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations + +### 1.0.0-rc.117 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add ROI results to the dataset info plots +* jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment + +### 1.0.0-rc.116 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add binning options in the context menu + +### 1.0.0-rc.115 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Default spot finding settings can be configured via config JSON +* jfjoch_viewer: FFT analysis of data in the dataset plot + +### 1.0.0-rc.114 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Fix generating JPEG images with resolution estimation + +### 1.0.0-rc.113 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Improve handling of rotation indexing +* jfjoch_broker: More information saved in CBOR end message (WIP) +* jfjoch_writer: Save rotation indexing lattice parameters and Niggli class +* jfjoch_viewer: Remove (for now) primitive cell information +* jfjoch_viewer: Use angle for dataset info plot for rotation scans + +### 1.0.0-rc.112 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Experimental rotation (3D) indexing +* jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values + +### 1.0.0-rc.111 +This is an UNSTABLE release. + +* jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment) +* jfjoch_viewer: Fix auto contrast not refreshing image + +### 1.0.0-rc.110 +This is an UNSTABLE release. + +* jfjoch_broker: Add auto-contrast option for preview images +* Frontend: Add logo image +* jfjoch_viewer: Add logo image +* jfjoch_viewer: For image chart allow to set min value to zero +* jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure +* jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there) +* Documentation: Add logo image + +### 1.0.0-rc.109 +This is an UNSTABLE release. + +* jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard +* jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors + +### 1.0.0-rc.108 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN) +* jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default +* jfjoch_viewer: Fit to window of image is now default when size of the grid changes + +### 1.0.0-rc.107 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor polishing of new functionality +* jfjoch_broker: User NaN for empty azimuthal bins + +### 1.0.0-rc.106 +This is an UNSTABLE release. + +* jfjoch_viewer: Allow for multiple dataset info plots +* jfjoch_viewer: Highlight current element in grid + +### 1.0.0-rc.105 +This is an UNSTABLE release. + +* jfjoch_viewer: Clean-up widgets slightly +* jfjoch_viewer: Limit right panel to 600 pixels +* jfjoch_viewer: Parse crystal symmetry type +* jfjoch_viewer: Grid scan view takes color map and can be fit to zoom + +### 1.0.0-rc.104 +This is an UNSTABLE release. + +* jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier) +* jfjoch_viewer: Display grid scan results in 2D (work in progress) +* jfjoch_viewer: Improve auto-scaling on start of images (work in progress) +* jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots + +### 1.0.0-rc.103 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer +* jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates + +### 1.0.0-rc.102 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer + +### 1.0.0-rc.101 +This is an UNSTABLE release. + +* jfjoch_viewer: Auto load is better handling change of states +* jfjoch_viewer: Fix DBus registration +* jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update +* jfjoch_viewer: Calculate ROI in a more efficient way + +### 1.0.0-rc.100 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix dbus registration +* jfjoch_viewer: Remove background slider for diffraction image +* jfjoch_viewer: Adjustments for 2D azimuthal image viewer + +### 1.0.0-rc.99 +This is an UNSTABLE release. + +* jfjoch_broker: Fix output during mask data collection + +### 1.0.0-rc.98 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization + +### 1.0.0-rc.97 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask +* jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems) +* jfjoch_viewer: Add option to handle user pixel mask (experimental) +* jfjoch_viewer: More options for ROI +* jfjoch_viewer: Add window to display calibration + +### 1.0.0-rc.96 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* jfjoch_broker: Remove PNG preview, no dependency on libpng +* jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds) +* jfjoch_viewer: Show data collection time in dataset tooltip +* jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate) +* jfjoch_viewer: Auto foreground value +* Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds +* Remove OpenBLAS dependency + +### 1.0.0-rc.95 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* Add git-lfs to Rocky8 docker image + +Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved. + +### 1.0.0-rc.94 +This is an UNSTABLE release. + +* FFTIndexer: Add limit on angles to avoid colinear vectors +* Docker images: Add 3D Qt +* Gitea: Fixes to the pipeline + +### 1.0.0-rc.93 +This is an UNSTABLE release. + +* CI: Fixes to Gitlab based pipeline +* PCIe driver: Fix PCIe revision being hex number + +### 1.0.0-rc.92 +This is an UNSTABLE release. + +* jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample. + +### 1.0.0-rc.91 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated +* jfjoch_broker: Spot finding and integration predictions are ported to a GPU +* jfjoch_broker: Estimate resolution +* jfjoch_broker: Lattice search +* jfjoch_broker: Many more improvements in image analysis + +### 1.0.0-rc.90 +This is an UNSTABLE release. + +* jfjoch_broker: for indexing min index spots for a viable cells can be changed via OpenAPI +* jfjoch_viewer: Optional auto-reanalyze images +* jfjoch_writer: Add option where no files at all are saved +* Documentation: improvements + +### 1.0.0-rc.89 +This is an UNSTABLE release. + +* jfjoch_broker: Fix resolution estimation code +* jfjoch_broker: Fix Wilson B-factor calculation code +* jfjoch_viewer: Improve display of plots +* jfjoch_viewer: Fix segmentation fault +* jfjoch_viewer: Display missing metadata when using HTTP +* jfjoch_viewer: Fix bug when opening the same file twice + +### 1.0.0-rc.88 +This is an UNSTABLE release. + +* jfjoch_viewer: Add resolution estimation to the image information +* jfjoch_broker: Minor changes to resolution estimate routine + +### 1.0.0-rc.87 +This is an UNSTABLE release. + +* jfjoch_viewer: Display more image metadata (angle / exposure time) +* jfjoch_viewer: Improve I/sigma and B-factor plots +* jfjoch_broker: Estimate resolution based on visible spots + +### 1.0.0-rc.86 +This is an UNSTABLE release. + +* jfjoch_broker: Update logic when initializing detector to make it a bit more resilient +* Gitea pipelines have nocuda option for all architectures + +### 1.0.0-rc.85 +This is an UNSTABLE release. + +* jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image +* jfjoch_viewer: Handle spot count better in dataset info plots +* jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled + +### 1.0.0-rc.84 +This is an UNSTABLE release. + +* jfjoch_broker: Write in log which detector is being initialized +* Changes to automated build system + +### 1.0.0-rc.83 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before) +* CI: Fix script to generate python client + +### 1.0.0-rc.82 +This is an UNSTABLE release. + +* jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment) +* Frontend: Minor fixes +* Build scripts: Minor fixes to FFTW + +### 1.0.0-rc.81 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing +* jfjoch_broker: Add FFTW based indexer for CPU only indexing +* jfjoch_broker: Enable saving X-ray fluorescence spectra +* jfjoch_writer: Write total spot count (before filtering) +* jfjoch_viewer: Add more information on source, sample, and buttom to show ice rings +* jfjoch_viewer: Enable data processing inside the viewer + +CI: Moving from Gitlab to Gitea at PSI + +### 1.0.0-rc.80 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer + +### 1.0.0-rc.79 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance +* jfjoch_writer: Save nimages_per_trigger in detectorSpecific + +### 1.0.0-rc.78 +This is an UNSTABLE release. + +* jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore) + +### 1.0.0-rc.77 +This is an UNSTABLE release. + +* jfjoch_viewer: Display detector and dataset settings with tooltips +* jfjoch_viewer: Clean excessive HDF5 warnings +* jfjoch_viewer: Display unit cell +* jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset + +### 1.0.0-rc.76 +This is an UNSTABLE release. + +* jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones +* jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median) +* jfjoch_writer: Save space group number (non-NXmx addition) in addition to name +* jfjoch_viewer: Fix the bug on reading space_group as string +* jfjoch_viewer: Add missing resolution labels on rings +* jfjoch_viewer: Remove Q value from the status bar + +### 1.0.0-rc.75 +This is an UNSTABLE release. + +* jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being + +### 1.0.0-rc.74 +This is an UNSTABLE release. + +* jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing) +* jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization + +### 1.0.0-rc.73 +This is an UNSTABLE release. + +* jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default + +### 1.0.0-rc.72 +This is an UNSTABLE release. +This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Refactor of indexing and geometry refinement code +* jfjoch_broker: Handle space group/centering in refinement code +* jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images +* jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion +* jfjoch_writer: Experimental saving of reflections +* jfjoch_writer: Save space group name as string +* jfjoch_viewer: Add profile radius and B-factor +* jfjoch_viewer: Show 4 digits for wavelength +* jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings) +* FPGA: Use UDP destination port to distinguish between detector modules and data streams +* FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up) +* FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene) + +### 1.0.0-rc.71 +This is an UNSTABLE release. + +* jfjoch_broker: Remove resolution estimation via machine learning +* jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers +* jfjoch_viewer: Fix error when HDF5 files with indexing results couldn't be loaded on a machine without GPU + +### 1.0.0-rc.70 +This is an UNSTABLE release. +This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement +* jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards) +* jfjoch_broker: Enable geometry refinement - first try (work in progress) +* jfjoch_viewer: Fix deadlock when opening HTTP connections +* jfjoch_viewer: Display rings as ellipses with detector tilt +* jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image +* jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3) + +* Add Google Ceres a non-linear least-square optimization library to Jungfraujoch +* Add experimental detector calibration routines (for LaB6) +* Improve documentation on the ZeroMQ writer notification socket and detector geometry + +### 1.0.0-rc.69 +This is an UNSTABLE release. + +* jfjoch_viewer: Metadata can be modified for an open dataset (no option to save) +* jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments +* jfjoch_viewer: Resolution rings not enabled by default +* jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file + +### 1.0.0-rc.68 +This is an UNSTABLE release. + +* jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector +* jfjoch_broker: Default detector settings can be configured for each detector separately +* jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings +* jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing +* jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter +* jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI + +### 1.0.0-rc.67 +This is an UNSTABLE release. + +* jfjoch_broker: Enable SSL +* jfjoch_broker: Wilson B-factor only provided is fit is relatively OK (R^2 > 0.3); this will be refined much more in the future + +### 1.0.0-rc.66 +This is an UNSTABLE release. + +* jfjoch_broker: Indexers operate as thread pool, which is operating +* jfjoch_viewer: Increase interval between loading images + fix too many verbose messages + +### 1.0.0-rc.65 +This is an UNSTABLE release. + +* jfjoch_broker: Print information regarding used image pushers +* jfjoch_viewer: Allow syncing with Jungfraujoch server +* OpenAPI: Clarify licensing terms in the file + +### 1.0.0-rc.64 +This is an UNSTABLE release. + +* jfjoch_broker: Fix issue in receiver light with very long preparation time for threads +* jfjoch_broker: Add verbose option +* jfjoch_broker: Don't trigger pedestal if critical settings are not changed when loading detector settings +* jfjoch_broker: Detector left in busy state when detector settings were improper +* jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times +* jfjoch_lite_perf_test: Add verbose option + +### 1.0.0-rc.63 +This is an UNSTABLE release. + +* jfjoch_broker: Save NX/NY for grid scan result +* jfjoch_broker: Add processing time to CBOR output and plot +* jfjoch_writer: Add processing time to data file + +### 1.0.0-rc.62 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where low resolution spots were not counted properly +* jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count +* jfjoch_broker: Add more spot count information to CBOR +* jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles + +### 1.0.0-rc.61 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector +* jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results +* jfjoch_broker: High resolution limit for low res. spot counting can be adjusted + +### 1.0.0-rc.60 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released +* jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices +* jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength + +### 1.0.0-rc.59 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock. +* jfjoch_writer: Verbose option for debugging. + +### 1.0.0-rc.58 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix memory leak +* jfjoch_writer: Add detector_number/serial_number to master file + +### 1.0.0-rc.57 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible +* jfjoch_viewer: "Movie" mode + +### 1.0.0-rc.56 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation + +### 1.0.0-rc.55 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation + +### 1.0.0-rc.54 +This is an UNSTABLE release. + +* jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not) +* jfjoch_broker: Don't remove shared memory area when deactivating detector +* jfjoch_writer: Save writer release +* jfjoch_viewer: Increase time for the messages in the status bar + +### 1.0.0-rc.53 +This is an UNSTABLE release. + +* PCIe driver: Imperfect solution for RHEL 9.5+ changes +* jfjoch_writer: Fix to angle containers for AutoProc compatibility +* jfjoch_fpga_test: Use consecutive number for devices, not interleaved + +### 1.0.0-rc.52 +This is an UNSTABLE release. + +* jfjoch_viewer: Use warmer colors to distinguish from AareGUI +* jfjoch_viewer: Minor adjustments to DBus setting image number +* jfjoch_broker: Fix in low resolution spot count plotting + +### 1.0.0-rc.51 +This is an UNSTABLE release. + +* jfjoch_broker: Send preview in PNG format +* jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range +* jfjoch_broker: Provide ML resolution estimation in scan result +* jfjoch_broker: Allow removing beam center in web preview + +### 1.0.0-rc.50 +This is an UNSTABLE release. + +* The release fixes some of many bugs introduced in recent releases +* jfjoch_viewer: display predictions for indexed cells + +### 1.0.0-rc.49 +This is an UNSTABLE release. + +* jfjoch_broker: Handle sample temperature (K) and ring current (mA) to metadata +* jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current + +### 1.0.0-rc.48 +This is an UNSTABLE release. + +* jfjoch_broker: fix the bug when a unit cell was not exported for a scan result. + +### 1.0.0-rc.47 +This is an UNSTABLE release. + +* jfjoch_viewer: fix dbus service path +* jfjoch_writer: fix CBF/TIFF writing + +### 1.0.0-rc.46 +This is an UNSTABLE release. + +* jfjoch_viewer: remove dependency on image analysis + +### 1.0.0-rc.45 +This is an UNSTABLE release. + +* jfjoch_broker: Detector list returns pixel size (mm) + +### 1.0.0-rc.44 +This is an UNSTABLE release. + +* jfjoch_broker: more general definition of scan result export + +Braking changes: +* It removes additions to OpenAPI from 1.0.0-rc.43 +* It makes changes to the "unit_cell" definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations. + +### 1.0.0-rc.43 +This is an UNSTABLE release. + +* jfjoch_broker: Export grid scan results into a single data structure + +### 1.0.0-rc.42 +This is an UNSTABLE release. + +* jfjoch_broker: Add pixel_sum to CBOR output. +* jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine +* jfjoch_writer: Save pixel_sum + +### 1.0.0-rc.41 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction +* jfjoch_broker: Export GPU count via OpenAPI +* jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector + +* FPGA: Increase the number of integration bins to 2048 + +### 1.0.0-rc.40 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures +* jfjoch_broker: Improve metadata for rotation data collection +* jfjoch_broker: Better handling of plotting +* jfjoch_broker: FFT based indexing +* jfjoch_broker: Integration, first try, results not saved at the moment +* jfjoch_broker: Internal improvements in image handling + +* jfjoch_writer: Multiple adjustments adapt to changes in this release for new features +* jfjoch_writer: New state management model to improve clarity of error reporting + +* jfjoch_viewer: Remote control via DBus + +* Frontend: Multiple adjustments for new features +* Frontend: Grid scan plots + +WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken. + +### 1.0.0-rc.39 +* FPGA: Bugfix for pixel masked for data analysis if summation was on +* jfjoch_viewer: Fix segmentation fault when cursor was outside of image + +### 1.0.0-rc.38 +* jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via RES +* jfjoch_broker: Neural net model can use all 4 quadrants of the detector +* jfjoch_broker: For EIGER image time can be provided through /start +* jfjoch_viewer: Add image list option +* jfjoch_viewer: Drawing circular ROIs with shift +* jfjoch_viewer: Enable image summation +* jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer + +### 1.0.0-rc.37 +* jfjoch_broker: Make locking rules more flexible +* jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors +* jfjoch_viewer: Add status bar + +### 1.0.0-rc.36 +This is UNSTABLE release. Wait for new version to use in a production environment. + +* jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code) +* jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation) +* jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data +* jfjoch_writer: Increase buffering to 1000 images in the receiver +* jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5 + +### 1.0.0-rc.35 +This is UNSTABLE release, not properly tested. Wait for new version for using production. + +* jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting. +* jfjoch_writer: Save EIGER energy threshold +* jfjoch_writer: Add `/entry/sample/goniometer` for compatibility with `eiger2cbf` program + +### 1.0.0-rc.34 +This is UNSTABLE release - introducing new features, but not properly tested. Wait for new version for using production. + +* jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings) +* jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells +* jfjoch_broker: Handle situation when detector doesn't want to gracefully stop (to be tested) +* jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots +* jfjoch_viewer: Can extract data analysis results from "legacy" format +* jfjoch_viewer: Display dataset name +* FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels +* FPGA: Add pixel sum to module statistics +* FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI +* FPGA: Spot finder is back to full dynamic range (24-bit) +* FPGA: More debug features for internal FIFOs + +Known issues: +* ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts. +* EIGER data acquisition has an issue that is currently debugged + +### 1.0.0-rc.33 +* jfjoch_broker: Fix issue with EIGER settings being loaded improperly + +### 1.0.0-rc.32 +* jfjoch_broker: Refactor code for azimuthal integration for further improvements +* jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later) +* jfjoch_writer: Fix too much verbose information +* FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20) + +### 1.0.0-rc.31 +This is UNSTABLE release - introducing many features, but still needs more testing. +Expecting soon to put bugfix release. + +* jfjoch_writer: Allow to enable overwriting existing files (not enabled by default) +* jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default) +* jfjoch_viewer: Image viewer work early test version +* jfjoch_broker: Fixes to counting packets per dataset/image +* jfjoch_broker: Image buffer is accessible for outside to check images +* jfjoch_broker: error/saturated pixels and dedicated ROI "beam" can be tracked online +* jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU +* jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation +* jfjoch_broker: Fix bug in EIGER initializing +* jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot +* PCIe driver: Add PCIe link speed and width +* FPGA: Improve counting error/saturated/min/max pixels +* FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas +* FPGA: Require Vivado 2022.2 + +Warning: +There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency. + +### 1.0.0-rc.30 +* jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to "loose" images within the writer + +### 1.0.0-rc.29 +* jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU +* jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file +* jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info) +* jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly + +Warning - two breaking changes to OpenAPI: +* Handling of ROIs is through `/config/roi` path only for both circle and box ROIs, path in `/roi` are no longer accessible +* `broker_status` structure introduced in 1.0.0-rc.28 has member `message` and not `error_message` to allow +handling info/warning messages as well + +### 1.0.0-rc.28 +* jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI +* jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not complaint to OpenAPI specs +* jfjoch_test: remove header that failed when CUDA is absent during compilation +* frontend: add soft trigger button in data collection tab +* frontend: show error message when in error state +* CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF) + +### 1.0.0-rc.27 +* jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation +* jfjoch_broker: update pistache web server version +* jfjoch_writer: minor changes to republish logic +* Improvements to documentation + +### 1.0.0-rc.26 +* jfjoch_broker: implement ZeroMQ stream for image metadata information +* jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent +* jfjoch_broker: add crystal lattice plots +* jfjoch_broker: remove empty bins from the plots +* jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU "long" summation +* jfjoch_broker: Fix bug when mean background estimation / indexing rate where affected by previous experiment +* jfjoch_writer: fix missing "-w" parameter +* jfjoch_writer: temporary files have ".tmp" suffix +* jfjoch_writer: refactor logic for watermarks +* jfjoch_writer: report on internal FIFO utilization +* jfjoch_writer: clean-up naming for azimuthal integration and background estimate +* jfjoch_writer: write final background estimate and indexing rate in the master file +* tools/: remove unnecessary tools, make naming consistent +* CBOR: Add indexing rate and background estimate to end message +* CBOR: Clean-up documentation + +### 1.0.0-rc.25 + +* Updates to documentation +* License set to GPLv3 / OHL-S +* Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms +* Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5 + +### 1.0.0-rc.24 + +New FPGA functionality: +* EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit "real" depth is 23-bit + 1-bit signed) +* Output possible to 8, 16 and 32-bit data +* Threshold is applied before summation +* Pixel mask can be applied on FPGA +* Mark pixels with ADC content = 0 as bad pixels +* FPGA stores semantic version information (access via /sys/class/misc/jfjoch.../version) + +New software functionality: +* Long summation (above 256 frames) done on CPU +* Mechanism to save arbitrary data to HDF5 file +* ZeroMQ preview has option to send start message +* Rework pixel mask + add statistics displayed in web interface + +Bug fixes: +* Web frontend: Update preview image automatically during data acquisition +* jfjoch_broker: Error handling if CUDA driver is not installed +* jfjoch_broker: Correctly update progress during pedestal +* jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF +* jfjoch_action_test: enable HLS simulation + +Documentation improvement and placement in a dedicated directory diff --git a/_sources/CPU_DATA_ANALYSIS.md.txt b/_sources/CPU_DATA_ANALYSIS.md.txt new file mode 100644 index 00000000..3e753c03 --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS.md.txt @@ -0,0 +1,616 @@ +# CPU-side crystallographic data analysis (Jungfraujoch) + +This document describes the crystallographic algorithms implemented in Jungfraujoch for **CPU**- and **GPU**-side real‑time and near‑real‑time data analysis. + +**Scope.** The pipeline covered here comprises: + +1. geometry mapping and corrections, +2. azimuthal integration (powder/radial profiles), +3. Bragg spot finding (strong pixels → connected components → spot descriptors), +4. indexing (still and rotation modes), +5. Bravais lattice / centering inference, +6. geometry and lattice refinement, +7. reflection prediction (still and rotation), +8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free), +9. scaling and merging, +10. merge-level error modelling and outlier rejection, +11. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref). + +## References + +The methods are inspired and reuising solutions implemented in: + +- W. Kabsch, “XDS”, *Acta Cryst.* **D66** (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts). +- W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, *Acta Cryst.* **D66** (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors). +- T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts). +- J. Kieffer & J. P. Wright, "PyFAI: a Python library for high performance azimuthal integration on GPU", *Powder Diffraction* **28** (2013), S339-S350 (detector geometry definition, azimuthal integration) +- H. Powell, "The Rossmann Fourier autoindexing algorithm in MOSFLM", *Acta Cryst.* **D55** (1999), 1690-1695 (FFT indexing) +(list is not exhaustive) + +## 1. Geometry, reciprocal-space mapping, and basic quantities + +### 1.1 Coordinate conventions + +For a pixel coordinate $(x,y)$ (in pixels), Jungfraujoch converts to a laboratory direction vector via: + +1. shift by direct-beam position $(x_\mathrm{beam}, y_\mathrm{beam})$, +2. scale by pixel size $p$ (mm), +3. set detector distance $D$ (mm), +4. apply detector orientation rotation $R_\mathrm{det}$ (PyFAI-like parameterization). + +The unnormalized detector coordinate (mm) is: +$ +\mathbf{r}_\mathrm{det}(x,y) = +\begin{pmatrix} +(x-x_\mathrm{beam})p\\ +(y-y_\mathrm{beam})p\\ +D +\end{pmatrix}. +$ + +The lab-frame vector is: +$ +\mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. +$ + +Let the incident wavevector magnitude be $k = 1/\lambda$ in Å$^{-1}$, and define: +$ +\mathbf{S}_0 = (0,0,k). +$ + +The **reciprocal-space scattering vector** associated with pixel $(x,y)$ is: +$ +\mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. +$ + +This $\mathbf{s}$ is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement. + +### 1.2 Two-theta, azimuth, resolution and $q$ + +The scattering angle $2\theta$ is computed from $\mathbf{r}_\mathrm{lab}$ via: +$ +2\theta = \arctan\!\left(\frac{\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2}}{z_\mathrm{lab}}\right). +$ + +Resolution (Å) at a pixel is: +$ +d = \frac{\lambda}{2\sin(\theta)} = \frac{\lambda}{2\sin(2\theta/2)}. +$ + +The magnitude $q = 2\pi/d$ is used for radial binning and ice-ring handling. + +### 1.3 Distance from the Ewald sphere + +For a reciprocal lattice point $\mathbf{p}$ (Å$^{-1}$), define: +$ +\Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. +$ +Jungfraujoch uses $|\Delta_\mathrm{Ewald}|$ as an operational proxy for excitation error. This appears in: +- still prediction (accept if $|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}$), +- profile radius estimation (see §11.1), +- still partiality option in scaling/merging (§10.2). + +--- + +## 2. Azimuthal integration (radial profiles) + +Azimuthal integration produces a radial profile $I(q)$ or $I(d)$ by histogramming pixels into radial bins. Pixels are **not split** across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 $\phi$ sectors (`azim_bins`, `--azim-phi-bins`), giving a **2D $q\times\phi$ profile** that exposes azimuthal anisotropy such as detector shadowing or sample texture. + +### 2.1 Histogram estimator + +Let bin index $b(x,y)$ be precomputed from $q(x,y)$ (or equivalently from $d(x,y)$) and, when $\phi$ sectors are enabled, the azimuth $\phi(x,y)$ — so $b = b_q + b_\phi B_q$. For each bin $b$: + +- accumulate corrected intensity and its square: + $ + S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad + S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, + $ +- and count: + $ + N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. + $ + +The profile reports both the mean $\bar{I}_b = S_b / N_b$ (when $N_b>0$) and a per-bin sample standard deviation $\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}$ (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded. + +### 2.2 Corrections applied + +Two standard corrections are available: + +**(i) Solid angle / geometric correction.** A flat pixel's solid angle falls off with the **incidence angle $\alpha$ between the scattered ray and the detector normal**. With the in-plane detector offsets $u=(x-x_\mathrm{beam})p$ and $v=(y-y_\mathrm{beam})p$ (§1.1) and detector distance $D$, +$ +\cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad +C_\Omega = \cos^3\alpha, +$ +applied — like the polarization term below — as a **divisor** (intensities are scaled by $1/\cos^3\alpha$), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because $\alpha$ is evaluated in the detector's own frame it is **invariant under detector tilt** ($\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}$), matching PyFAI's `solidAngleArray` and MAX IV azint. It reduces to the commonly quoted $\cos^3(2\theta)$ form only for an untilted detector, where the incidence angle coincides with the scattering angle. + +**(ii) Polarization correction.** With polarization coefficient $P$ (beamline dependent) and azimuth $\phi$: +$ +C_\mathrm{pol}(2\theta,\phi) = +\frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), +$ +applied as a divisor to intensities (i.e. scale by $1/C_\mathrm{pol}$) when enabled. + +### 2.3 Background estimate for profiles + +A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid $Q$ window (default $2\pi/5$ to $2\pi/3$ Å$^{-1}$). This background is used for monitoring and diagnostics; it is **not** the same as the local Bragg-spot background used in summation integration (§9.2). + +--- + +## 3. Spot finding (strong pixels → Bragg spots) + +Spot finding is a two-stage process: + +1. **Strong-pixel selection** using intensity and/or local signal-to-noise criteria. +2. **Connected-component labeling (CCL)** to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction. + +### 3.1 Strong-pixel detection by local statistics + +For each pixel $i$ with value $v_i$, consider a square window (nominally $31\times 31$ pixels) around it. Let the window contain $n$ valid pixels (excluding masked/bad/saturated), and define: +$ +\Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. +$ + +To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: +$ +\Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. +$ + +A variance-like quantity proportional to $n'^2$ is formed: +$ +V = n'\Sigma_2' - (\Sigma')^2, +$ +and the deviation-from-mean quantity: +$ +\Delta = v_i n' - \Sigma'. +$ + +A pixel is considered strong if: +- it is above a photon/count threshold, and +- its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and +- $\Delta>0$, and +- the squared deviation exceeds a scaled variance: + $ + \Delta^2 > V\cdot T^2, + $ + where $T$ is the configured signal-to-noise threshold. + +This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast. + +Special cases: +- saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots), +- invalid pixels are never strong. + +### 3.2 Resolution and ice-ring handling + +Spot finding can be restricted to a resolution range $[d_\mathrm{high}, d_\mathrm{low}]$ by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6). + +A single per-image **ice-ring score** is derived from the azimuthally-integrated radial profile: for each hexagonal-ice powder ring (positions $d$ from Moreau *et al.*, Acta Cryst D77, 2021), the profile intensity at the ring is divided by a smooth background estimated from the *whole* profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring's ratio is reported (1 = no ice, $>1$ = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring's own edge (offline processing defaults to a fine 0.01 1/Å spacing, `--azim-q-spacing`, so the rings are well resolved). (A significance/z-score was considered but is uninformative here: with many photons any real ice ring is highly significant, so the discriminating quantity is the ice *magnitude*, i.e. this ratio.) It is stored per image (`ice_ring_score`, HDF5 `/entry/MX/iceRingScore`) as a monitoring quantity, distinct from the merge-time ice masking, which is data-driven from the per-ring merged CC1/2. + +A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in $1/d$ (or $q$) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low. + +### 3.3 Connected-component labeling (CCL) + +Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with: + +- centroid $(x,y)$ (often intensity-weighted), +- pixel count (spot size), +- integrated spot intensity proxy (sum of pixel values), +- resolution $d$ at the centroid (or mean over pixels), +- and quality flags (e.g. ice-ring classification). + +Spot-level filters include minimum/maximum pixel count and resolution limits. + +--- + +## 4. Indexing overview + +Indexing maps observed reciprocal-space vectors $\mathbf{s}_i$ to a lattice such that: +$ +\mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, +$ +with integer $(h_i,k_i,l_i)$. + +Jungfraujoch supports two complementary indexing strategies: + +1. **FFT-based indexing** (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples. +2. **Fast-feedback indexing** (TORO-like): requires an approximate unit cell; optimized for speed and feedback. + +Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return **more than one lattice** per image (multi-lattice indexing; see §5.4). + +### 4.1 Indexed-spot decision (inlier test) + +Given a trial lattice with direct basis vectors $\mathbf{a},\mathbf{b},\mathbf{c}$ (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: +$ +h_f = \mathbf{s}\cdot\mathbf{a},\quad +k_f = \mathbf{s}\cdot\mathbf{b},\quad +l_f = \mathbf{s}\cdot\mathbf{c}. +$ +Let $(h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))$ and define the fractional residual: +$ +\delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. +$ +A spot is indexed if $\delta^2 < \tau^2$, where $\tau$ is the configured tolerance. + +For indexed spots, the reciprocal lattice point $\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*$ is used to compute $\Delta_\mathrm{Ewald}(\mathbf{p})$ (stored as a diagnostic and later used in profile-radius estimation). + +--- + +## 5. FFT indexing (unknown unit cell) + +FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms. + +### 5.1 Directional projections and histograms + +Choose a set of unit vectors $\{\mathbf{u}_d\}$ on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction $d$, form a histogram in the scalar projection: +$ +t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. +$ + +Bin width is chosen approximately as: +$ +\Delta t \approx \frac{1}{2 L_\mathrm{max}}, +$ +where $L_\mathrm{max}$ is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum $q$ used (set by a high-resolution cutoff for indexing). + +### 5.2 FFT peak picking and candidate vectors + +For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along $\mathbf{u}_d$. Each direction yields a candidate real-space length $L$ chosen **not** by raw magnitude but by **maximum prominence above a running-mean local background** (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to $L\ge L_\mathrm{min}$. + +Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. + +A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak. + +### 5.3 Lattice reduction and cell candidates + +Triples of candidate vectors are combined to form candidate bases $(\mathbf{A},\mathbf{B},\mathbf{C})$, each reduced to its **Niggli-reduced cell** (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a **widened fallback** anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses. + +### 5.4 Robust refinement and best-cell selection + +Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked: +1. more indexed spots wins — **unless** two candidates index within ~10 % of each other, in which case +2. the **smaller-volume** cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then +3. the smaller refinement score, then the spot count again. + +Selection is **not limited to a single lattice**: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded. + +An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant). + +--- + +## 6. Bravais lattice / centering inference (“lattice search”) + +If the space group is supplied by the user, its lattice constraints are assumed for refinement and subsequent processing. + +If not, Jungfraujoch attempts to infer the most plausible Bravais lattice type from the metric tensor after Niggli reduction: + +1. **Niggli reduction** is performed to obtain a reduced cell in $G^6$ representation (Gruber vector). +2. The reduced cell is compared against a list of Niggli classes corresponding to Bravais lattices and centerings. +3. The highest-symmetry class that matches within tolerances is selected (relative metric tolerance and angular tolerance). + +The output includes: +- a conventional cell, +- crystal system (triclinic, monoclinic, …), +- centering symbol (one of $P, C, I, F, R$; the $A/B$ variants are not emitted here — they are handled only later as prediction absences, §8.4). + +This stage provides centering information used for systematic absences in prediction (§8.4) and for reporting. + +**Note.** In ambiguous or special cases, forcing space group to $P1$ (no symmetry assumptions) is recommended. + +--- + +## 7. Geometry and lattice refinement + +Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices. + +### 7.1 Parameterization + +The refinement jointly optimizes, depending on mode and constraints: + +- beam center $(x_\mathrm{beam}, y_\mathrm{beam})$, +- detector distance $D$, +- detector tilt angles (two-angle model; third rotation often held at 0), +- rotation axis direction (for rotation datasets), +- crystal orientation (a global rotation), +- unit-cell parameters, with constraints determined by inferred crystal system. + +By default only the beam center, unit cell and crystal orientation are refined; the detector distance, tilt angles and rotation-axis direction are held fixed unless explicitly enabled. A lighter **orientation-only** mode refines just the crystal orientation (with a weak small-rotation prior on the poorly-determined out-of-plane component), for stills whose geometry is already trusted. + +For higher symmetries, constraints are enforced, e.g. +- cubic: $a=b=c,\ \alpha=\beta=\gamma=90^\circ$, +- tetragonal: $a=b$, +- hexagonal: $a=b,\ \gamma=120^\circ$, +- monoclinic (unique axis $b$): $\alpha=\gamma=90^\circ$, $\beta$ refined. + +### 7.2 Residuals and objective + +For each indexed spot assigned integer $(h,k,l)$, compute: + +- observed reciprocal vector $\mathbf{s}_\mathrm{obs}$ from its detector position and current geometry, +- predicted reciprocal vector $\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})$. + +Residual is: +$ +\mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. +$ + +A non-linear least squares solver minimizes $\sum \|\mathbf{r}\|^2$ over all selected inlier spots. + +### 7.3 Rotation datasets: bringing observations to a common reference frame + +For oscillation/rotation data, each image corresponds to a rotation angle $\phi$ about an axis $\mathbf{m}_2$. Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: +$ +\mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, +$ +with $R(\phi)$ constructed from the axis-angle representation of the goniometer model. The angle $\phi$ is taken at the centre of each frame's oscillation (the frame angle plus half the oscillation width). + +### 7.4 Multi-stage tightening of inlier tolerance + +Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance $0.3\to0.2\to0.1$), which stabilizes convergence when starting from imperfect indexing and approximate geometry. + +--- + +## 8. Reflection prediction + +Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model. + +### 8.1 Enumerating reciprocal lattice points + +For a maximum resolution $d_\mathrm{min}$, accept $(h,k,l)$ such that: +$ +\lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. +$ + +### 8.2 Still prediction (excitation-error cutoff) + +For still images, the diffracting condition is approximated by an excitation-error cutoff: +$ +\left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. +$ +Accepted reflections are projected to the detector by intersecting the diffracted direction $\mathbf{S}=\mathbf{S}_0+\mathbf{p}$ with the detector plane, using the current geometry. + +When the beam has a finite energy bandwidth, this window is **broadened radially per reflection**: the cutoff is combined in quadrature with a bandwidth smear, $\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}$, where $\sigma_\mathrm{bw}\propto|p_z|$ (the reciprocal-space depth along the beam, growing as $\sim 1/d^2$). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same $\sigma_\mathrm{bw}$ is deconvolved from the measured profile radius (§11.1), so it is not double-counted. + +### 8.3 Rotation prediction (Laue equation + partiality model) + +For rotation/oscillation datasets, Jungfraujoch solves for rotation angles $\phi$ where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define: + +- rotation axis unit vector $\mathbf{m}_2$, +- $\mathbf{S}_0$ incident vector, +- $\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)$. + +A key quantity is: +$ +\zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad +\mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, +$ +which also appears in XDS as the Lorentz component linked to the rotation axis. + +A Gaussian mosaicity model yields a partiality fraction over an oscillation width $\Delta\phi$: + +$ P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], $ + +with mosaicity $\sigma_M$ in radians. + +Reflections are predicted if they meet minimum $\zeta$ and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area. + +### 8.4 Systematic absences (centering) + +Systematic absences are applied at least at the centering level (prior to full space-group symmetry). For centering symbol $C$: + +- $I$: absent if $h+k+l$ odd, +- $A$: absent if $k+l$ odd, +- $B$: absent if $h+l$ odd, +- $C$: absent if $h+k$ odd, +- $F$: absent if any of $h+k, h+l, k+l$ is odd, +- $R$: absent if $(-h+k+l)\bmod 3 \ne 0$, +- $P$: no centering absences. + +--- + +## 9. 2D Bragg integration (profile fitting over a three-ring ROI) + +Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The **default** extraction is **profile fitting** (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection $(I,\sigma,\text{partiality},d)$, so scaling, the rotation combine (§10.6) and merging consume either unchanged. + +### 9.1 Regions of interest + +For each predicted reflection at $(x_p,y_p)$, define three radii: + +- $r_1$: inner signal radius, +- $r_2$: inner background radius, +- $r_3$: outer background radius. + +Pixels are classified by their squared distance $r^2=(x-x_p)^2+(y-y_p)^2$: + +- **signal region:** $r^2 < r_1^2$, +- **background annulus:** $r_2^2 \le r^2 < r_3^2$. + +Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk ($r-1.el8.x86_64.rpm` that needs to be installed and contains all the necessary software and web interface. + +On other OSes one needs to compile Jungfraujoch from source (from the repo directory): +``` +$ mkdir build +$ cd build +$ cmake .. -DCMAKE_INSTALL_PREFIX= +$ make +$ sudo make install +``` +For manual installation, we recommend to use non-standard directory (like `/opt/jfjoch`), to facilitate upgrades and removal. +For DKMS to manage kernel module sources it is necessary to copy driver sources to `/usr/src/jfjoch-` directory. This requires extra flag in cmake `-DJFJOCH_INSTALL_DRIVER_SOURCE=ON`. + +Frontend web user interface has to be built separately with: +``` +$ cd build +$ make frontend +``` +Frontend files (.html and .js) will be placed in `frontend/dist` (outside of `build/` directory!) and has to be copied to a general location, e.g. `/usr/local/jfjoch/frontend` or `/opt/jfjoch/frotend`. + +## Flash the U55C FPGA card with a proper image and install Linux kernel driver. + +### Firmware flashing +1. Check that the card is detected by OS with "lspci |grep Xilinx" and check the PCIe bus/device/function (BDF) number, `11:00.0` in this case: +``` +$ lspci |grep Xilinx +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2) +``` +Note the device number `3450` that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and `rev 2` identifying release of the firmware. + +2. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given): +``` +$ sudo lspci -vv -s +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 +(...) +LnkSta: Speed 16GT/s (ok), Width x8 (ok) +(...) +``` + +3. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn't allways reach correct timing). +4. Flash the card with `xbflash.qspi` tool (part of Jungfraujoch). For fresh card use: +``` +sudo xbflash.qspi --primary --card --bar-offset 0x1f06000 +``` +For card that was already flashed with Jungfraujoch images: + +``` +sudo xbflash.qspi --primary --card +``` +It is necessary to confirm the operation by pressing `Y` key or one can add `--force` option to avoid confirmation. +It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions. + +5. Cold reboot: +``` +sudo ipmitool chassis power cycle +``` + +### Install PCIe driver + +For first run it is though recommended to try the driver without installing to the kernel directory: +``` +$ cd fpga/pcie_driver +$ make +$ sudo insmod jfjoch.ko +``` + +Check with `dmesg` that the device was properly found: +``` +$ dmesg |grep jfjoch +[ 431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142) +[ 431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a +``` + +If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. +On RHEL 8 you can install prebuilt RPM provided in the Gitlab package registry. On other systems follow procedure in +[PCIe driver](FPGA_PCIE_DRIVER.md). + +NOTE: Driver installation procedure on non-RHEL 8 systems is not well understood/optimized at the moment. + +NOTE: In case driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk if driver is updated: +``` +$ sudo dracut -f +``` +### Configure network +Configure switch according to [FPGA network guide](FPGA_NETWORK.md) - specifically set manual speed and turn off auto-negotiation +for the port used to connect U55C card and connect card to switch. + +### Running Jungfraujoch software +Main Jungfraujoch service is called `jfjoch_broker`. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. +It is recommended to run the service as `systemd` service. + +`jfjoch_broker` takes two parameters: JSON configuration file and HTTP port (default is 5232). +Example JSON files are placed in `etc/` folder. JSON file format is also explained in the OpenAPI definition, as `jfjoch_settings` data structure. + +When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring. + +Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one `jfjoch_broker` on a single machine, each confined to a disjoint subset of GPUs, set `CUDA_VISIBLE_DEVICES`; setting `CUDA_DEVICE_ORDER=PCI_BUS_ID` keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host: +``` +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232 +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233 +``` + +To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. +These need to be obtained from the PSI Detector Group. + +### Card verification + +To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_fpga_test` tool. +For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs: +``` +jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 +``` +Or 1M pixel system with one FPGA card: +``` +jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000 +``` + +## Install Jungfraujoch writer +Jungfraujoch writer is an additional service, that can connect to `jfjoch_broker` ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard. + +At the moment it is better to have a separate machine, with access to distributed file system, for writing images. + +Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands: +``` +mkdir build +cd build +cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch +``` + +## Install Jungfraujoch image viewer +Jungfraujoch viewer is X-ray diffraction image viewer, that is optimized to open Jungfraujoch HDF5 files. + +The viewer is a Qt application and it requires recent version of the library, therefore it is an optional dependency. + +To include it in the building of Jungfraujoch use `-DJFJOCH_VIEWER_BUILD=ON` directive for CMake: +``` +mkdir build +cd build +cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch +``` + + +## Install Jungfraujoch Python client +Use pip: +```shell +pip install jfjoch-client +``` \ No newline at end of file diff --git a/_sources/DETECTORS.md.txt b/_sources/DETECTORS.md.txt new file mode 100644 index 00000000..4e817fe3 --- /dev/null +++ b/_sources/DETECTORS.md.txt @@ -0,0 +1,13 @@ +# Supported detectors + +## PSI detectors +Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfruajoch controls the detector via statically compiled `slsDetectorPackage` into its source code. +It is important that detector firmware has to match `slsDetectorPackage` version used in Jungfraujoch (8.0.2 at the moment). +See [PSI Detector group website](https://www.psi.ch/en/lxn/software-releases) for details. + +# DECTRIS detectors + +Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. +In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles output data stream of the DCU. +This mode, called "lite" mode, doesn't use FPGA boards, but mostly CPUs and GPUs for indexing. +The mode is currently experimental and intended for low data rates (100 Hz). diff --git a/_sources/DETECTOR_GEOMETRY.md.txt b/_sources/DETECTOR_GEOMETRY.md.txt new file mode 100644 index 00000000..15534556 --- /dev/null +++ b/_sources/DETECTOR_GEOMETRY.md.txt @@ -0,0 +1,24 @@ +# Detector geometry + +At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location +vs. detector frame. It is not recommended to place detector modules stacked. + +The simplest case is detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance +and wavelength. + +For more complex case, one can provide tilt of the detector rotation in PyFAI convention. +This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, +where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector +is defined with three rotation angles: `rot1` (rotating detector right), `rot2` (rotating detector downwards), `rot3` (rotating detector clockwise). +See [PyFAI documentation](https://pyfai.readthedocs.io/en/stable/) for more details. + +## Macromolecular crystallography convention for the vertical direction +One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, +with Y values increasing downwards. This is also consistent with computer image formats. + +However, other techniques (as well as internal operation of PSI X-ray detectors) might follow convention, for point (0,0) +being in the bottom left corner and Y values increasing upwards. Such a convention is used, for example, by PyFAI. + +In general, convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows mirroring detector in Y. + +Extra care has to be taken by the user to ensure that no errors are made. \ No newline at end of file diff --git a/_sources/FPGA.md.txt b/_sources/FPGA.md.txt new file mode 100644 index 00000000..61a4fcc7 --- /dev/null +++ b/_sources/FPGA.md.txt @@ -0,0 +1,81 @@ +# FPGA smartNIC + +See separate document for [installation instructions](DEPLOYMENT.md). + +## Hardware +Currently supported FPGA is only **Xilinx Alveo U55C**. + +See AMD/Xilinx webpage for [card user guide (UG1469)](https://docs.xilinx.com/r/en-US/ug1469-alveo-u55c). +According to the user guide: +``` +Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling. +``` + +Card needs to be placed in PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. +There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. +Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C. + +Two variants of the card are available: +* `100g` - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch. +* `8x10g` - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables +are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch. + +See [network documentation](FPGA_NETWORK.md) for details of network. + +## Building firmware +Xilinx Vivado version has to precisely match version described in [the system requirements](../README.md. +only when `vivado` and `vitis_hls` are detected in the path. + +### Xilinx Vivado +The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version **2022.2** installed on the machine. +Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - +specifically newer versions of Vivado will not work. + +In additional to Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary: +* Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html), to build `100g` design. +* Paid 10G/25G Subsystem for Ultrascale+ to build `8x10g` design. +PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams +generated by PSI continuous integration pipeline for `8x10g` is only allowed for non-commercial use. +### HLS compilation +Make HLS routines: +``` +mkdir build +cd build +cmake .. +make hls +``` + +### Synthesis +Create PCIe `100g` bitstream with the following command: +``` +mkdir build +cd build +cmake .. +make pcie_100g +``` +and `8x10g`: +``` +mkdir build +cd build +cmake .. +make pcie_8x10g +``` +### When Vivado is not present + +During CMake execution, the following executables: `vivado` and `vitis_hls` must be present in the path. +If not, build targets will not be generated, and such or similar error message will show up: +``` +$ make pcie_100g +make: *** No rule to make target 'pcie_100g'. Stop. +``` + +### Gitlab CI +If Gitlab CI is properly set-up, firmware will be automatically built for every commit that starts with FPGA. +Built firmware should be downloaded as MCS files. + +### Frame generator + +Jungfraujoch card is equipped with frame generator. It allows to simulate JUNGFRAU detector without having access to such system. +It is placed in parallel to Ethernet MAC - so it is placed before the network stack and before any processing happening on the card. +In the future a redirection will be possible to send the simulated stream through the 100G TX network link. +Frame generator is written in HLS and controlled with AXI-Lite. \ No newline at end of file diff --git a/_sources/FPGA_DATA_ANALYSIS.md.txt b/_sources/FPGA_DATA_ANALYSIS.md.txt new file mode 100644 index 00000000..772fd84f --- /dev/null +++ b/_sources/FPGA_DATA_ANALYSIS.md.txt @@ -0,0 +1,82 @@ +# FPGA data analysis + +Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities. + +## Pixel mask +Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. +According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) +and will be excluded from a subsequent analysis. + +## ADU histogram +Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature +of unconverted values. This is done on a module-basis and works with bins with 32 ADU width. + +For EIGER this can be used as just a histogram procedure. + +## JUNGFRAU conversion +For JUNGFRAU module images are converted from ADUs to energy value and divided by a given number to keV units. +Result of the operation is rounded to integers. + +## Pixel thresholding +Pixel range can be specified. +Pixels below a minimum threshold will be assigned zero. +Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). +This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled. + +## Frame summation +Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together. + +## Azimuthal integration +To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. +This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. +Corrections were implemented according to formulas developed by [Jensen et al. (J. Synchr. Rad., 29, 1420-1428, 2022)](https://journals.iucr.org/s/issues/2022/06/00/fv5148/). + +Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited as 1024 per detector module. +This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. +One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit pixed point integer implementation +and is tuned for standard SAXS/WAXS range. + +As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, +so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows. + +## Spot finding +Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels: +1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run). +2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding, +3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation, +4. Pixel value - pixels above certain threshold value can be marked as strong, +5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong, +6. Connected pixels - strong pixels can be discarded if they are "alone", so their 8 directly neighboring pixels are not counted as strong pixels. + +While besides bad pixels criterion, all the above are optional (can be turned off), only pixels that fulfill all enabled criteria are selected as strong pixels. + +### SNR ratio calculation +Signal-to-noise ratio is calculated for a rectangular area. +In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. +This is dictated by the data flow within the FPGA. +In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. +Given very large box size, approximation are made, for example that `N ≈ N-1` in calculating standard deviation. + +## Region-of-interest (ROI) integration +Each pixel in a module can be assigned to one of 64 ROIs. For each ROIs, sum, sum of squares, max count, and number of valid pixels will be calculated. +Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis. + +ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal +(Q-range with an optional φ-sector) — are also evaluated on CPU by the shared `image_analysis/roi/` +engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for +detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path. + +## Pixel statisitics +The following statistics are collected for each module: +* Number of masked pixels +* Number of saturated pixels (excl.masked) +* Number of error pixels (excl. masked) +* Sum of valid pixels in the module +* Minimum value of valid pixels in the module +* Maximum value of valid pixels in the module +Valid pixels are not masked, not saturated, not error pixels. + +## Square root compression +Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. +Scheme was described in [Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020](https://doi.org/10.1107/S1600576720001776). +Pixel value `X` is replaced with `sqrt(N*X)`, where `N` is integer constant in range 1 to 16. diff --git a/_sources/FPGA_DESIGN.md.txt b/_sources/FPGA_DESIGN.md.txt new file mode 100644 index 00000000..56cd1c19 --- /dev/null +++ b/_sources/FPGA_DESIGN.md.txt @@ -0,0 +1,21 @@ +# FPGA data flow + +The following steps are performed on FPGA (in the order of operation): + +1. UDP header decoding +2. SLS detector header decoding +3. State machine that controls data acquisition (start/stop/cancel) +4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules +5. ADU histogram for JUNGFRAU +6. Mask pixels from missing packets with special value +7. Reorder lines for EIGER to form a proper module +8. Mask pixels based on provided pixel mask +9. JUNGFRAU conversion with gain and pedestal corrections +10. Threshold to zero pixels below certain count value +11. Integration according to predefined map (e.g., 1D azimuthal integration) +12. Spot finding +13. ROI calculation +14. Image lossy compression using N*sqrt(pixel) values +15. Send images, analysis results and metadata to host memory via PCI Express + +Each step has dedicated core, written in the high-level synthesis. Exact operation of cores for data analysis is explained in dedicated [document](FPGA_DATA_ANALYSIS.md). \ No newline at end of file diff --git a/_sources/FPGA_LICENSE.md.txt b/_sources/FPGA_LICENSE.md.txt new file mode 100644 index 00000000..540a9605 --- /dev/null +++ b/_sources/FPGA_LICENSE.md.txt @@ -0,0 +1,295 @@ +# FPGA license + +FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. +The license is equivalent of GNU Public License with adaptations for hardware. +See [OHL webpage](https://ohwr.org/project/cernohl/-/wikis/Documents/CERN-OHL-version-2) for details and FAQs. + +## CERN Open Hardware Licence Version 2 - Strongly Reciprocal + + +Preamble + +CERN has developed this licence to promote collaboration among +hardware designers and to provide a legal tool which supports the +freedom to use, study, modify, share and distribute hardware designs +and products based on those designs. Version 2 of the CERN Open +Hardware Licence comes in three variants: CERN-OHL-P (permissive); and +two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this +licence, CERN-OHL-S (strongly reciprocal). + +The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in +unmodified form only. + +Use of this Licence does not imply any endorsement by CERN of any +Licensor or their designs nor does it imply any involvement by CERN in +their development. + + +1 Definitions + +1.1 'Licence' means this CERN-OHL-S. + +1.2 'Compatible Licence' means + +a) any earlier version of the CERN Open Hardware licence, or + +b) any version of the CERN-OHL-S, or + +c) any licence which permits You to treat the Source to which + it applies as licensed under CERN-OHL-S provided that on + Conveyance of any such Source, or any associated Product You + treat the Source in question as being licensed under + CERN-OHL-S. + +1.3 'Source' means information such as design materials or digital +code which can be applied to Make or test a Product or to +prepare a Product for use, Conveyance or sale, regardless of its +medium or how it is expressed. It may include Notices. + +1.4 'Covered Source' means Source that is explicitly made available +under this Licence. + +1.5 'Product' means any device, component, work or physical object, +whether in finished or intermediate form, arising from the use, +application or processing of Covered Source. + +1.6 'Make' means to create or configure something, whether by +manufacture, assembly, compiling, loading or applying Covered +Source or another Product or otherwise. + +1.7 'Available Component' means any part, sub-assembly, library or +code which: + +a) is licensed to You as Complete Source under a Compatible + Licence; or + +b) is available, at the time a Product or the Source containing + it is first Conveyed, to You and any other prospective + licensees + +i) as a physical part with sufficient rights and + information (including any configuration and + programming files and information about its + characteristics and interfaces) to enable it either to + be Made itself, or to be sourced and used to Make the + Product; or +ii) as part of the normal distribution of a tool used to + design or Make the Product. + +1.8 'Complete Source' means the set of all Source necessary to Make +a Product, in the preferred form for making modifications, +including necessary installation and interfacing information +both for the Product, and for any included Available Components. +If the format is proprietary, it must also be made available in +a format (if the proprietary tool can create it) which is +viewable with a tool available to potential licensees and +licensed under a licence approved by the Free Software +Foundation or the Open Source Initiative. Complete Source need +not include the Source of any Available Component, provided that +You include in the Complete Source sufficient information to +enable a recipient to Make or source and use the Available +Component to Make the Product. + +1.9 'Source Location' means a location where a Licensor has placed +Covered Source, and which that Licensor reasonably believes will +remain easily accessible for at least three years for anyone to +obtain a digital copy. + +1.10 'Notice' means copyright, acknowledgement and trademark notices, +Source Location references, modification notices (subsection +3.3(b)) and all notices that refer to this Licence and to the +disclaimer of warranties that are included in the Covered +Source. + +1.11 'Licensee' or 'You' means any person exercising rights under +this Licence. + +1.12 'Licensor' means a natural or legal person who creates or +modifies Covered Source. A person may be a Licensee and a +Licensor at the same time. + +1.13 'Convey' means to communicate to the public or distribute. + + +2 Applicability + +2.1 This Licence governs the use, copying, modification, Conveying +of Covered Source and Products, and the Making of Products. By +exercising any right granted under this Licence, You irrevocably +accept these terms and conditions. + +2.2 This Licence is granted by the Licensor directly to You, and +shall apply worldwide and without limitation in time. + +2.3 You shall not attempt to restrict by contract or otherwise the +rights granted under this Licence to other Licensees. + +2.4 This Licence is not intended to restrict fair use, fair dealing, +or any other similar right. + + +3 Copying, Modifying and Conveying Covered Source + +3.1 You may copy and Convey verbatim copies of Covered Source, in +any medium, provided You retain all Notices. + +3.2 You may modify Covered Source, other than Notices, provided that +You irrevocably undertake to make that modified Covered Source +available from a Source Location should You Convey a Product in +circumstances where the recipient does not otherwise receive a +copy of the modified Covered Source. In each case subsection 3.3 +shall apply. + + You may only delete Notices if they are no longer applicable to + the corresponding Covered Source as modified by You and You may + add additional Notices applicable to Your modifications. + Including Covered Source in a larger work is modifying the + Covered Source, and the larger work becomes modified Covered + Source. + +3.3 You may Convey modified Covered Source (with the effect that You +shall also become a Licensor) provided that You: + +a) retain Notices as required in subsection 3.2; + +b) add a Notice to the modified Covered Source stating that You + have modified it, with the date and brief description of how + You have modified it; + +c) add a Source Location Notice for the modified Covered Source + if You Convey in circumstances where the recipient does not + otherwise receive a copy of the modified Covered Source; and + +d) license the modified Covered Source under the terms and + conditions of this Licence (or, as set out in subsection + 8.3, a later version, if permitted by the licence of the + original Covered Source). Such modified Covered Source must + be licensed as a whole, but excluding Available Components + contained in it, which remain licensed under their own + applicable licences. + + +4 Making and Conveying Products + +You may Make Products, and/or Convey them, provided that You either +provide each recipient with a copy of the Complete Source or ensure +that each recipient is notified of the Source Location of the Complete +Source. That Complete Source is Covered Source, and You must +accordingly satisfy Your obligations set out in subsection 3.3. If +specified in a Notice, the Product must visibly and securely display +the Source Location on it or its packaging or documentation in the +manner specified in that Notice. + + +5 Research and Development + +You may Convey Covered Source, modified Covered Source or Products to +a legal entity carrying out development, testing or quality assurance +work on Your behalf provided that the work is performed on terms which +prevent the entity from both using the Source or Products for its own +internal purposes and Conveying the Source or Products or any +modifications to them to any person other than You. Any modifications +made by the entity shall be deemed to be made by You pursuant to +subsection 3.2. + + +6 DISCLAIMER AND LIABILITY + +6.1 DISCLAIMER OF WARRANTY -- The Covered Source and any Products +are provided 'as is' and any express or implied warranties, +including, but not limited to, implied warranties of +merchantability, of satisfactory quality, non-infringement of +third party rights, and fitness for a particular purpose or use +are disclaimed in respect of any Source or Product to the +maximum extent permitted by law. The Licensor makes no +representation that any Source or Product does not or will not +infringe any patent, copyright, trade secret or other +proprietary right. The entire risk as to the use, quality, and +performance of any Source or Product shall be with You and not +the Licensor. This disclaimer of warranty is an essential part +of this Licence and a condition for the grant of any rights +granted under this Licence. + +6.2 EXCLUSION AND LIMITATION OF LIABILITY -- The Licensor shall, to +the maximum extent permitted by law, have no liability for +direct, indirect, special, incidental, consequential, exemplary, +punitive or other damages of any character including, without +limitation, procurement of substitute goods or services, loss of +use, data or profits, or business interruption, however caused +and on any theory of contract, warranty, tort (including +negligence), product liability or otherwise, arising in any way +in relation to the Covered Source, modified Covered Source +and/or the Making or Conveyance of a Product, even if advised of +the possibility of such damages, and You shall hold the +Licensor(s) free and harmless from any liability, costs, +damages, fees and expenses, including claims by third parties, +in relation to such use. + + +7 Patents + +7.1 Subject to the terms and conditions of this Licence, each +Licensor hereby grants to You a perpetual, worldwide, +non-exclusive, no-charge, royalty-free, irrevocable (except as +stated in subsections 7.2 and 8.4) patent licence to Make, have +Made, use, offer to sell, sell, import, and otherwise transfer +the Covered Source and Products, where such licence applies only +to those patent claims licensable by such Licensor that are +necessarily infringed by exercising rights under the Covered +Source as Conveyed by that Licensor. + +7.2 If You institute patent litigation against any entity (including +a cross-claim or counterclaim in a lawsuit) alleging that the +Covered Source or a Product constitutes direct or contributory +patent infringement, or You seek any declaration that a patent +licensed to You under this Licence is invalid or unenforceable +then any rights granted to You under this Licence shall +terminate as of the date such process is initiated. + + +8 General + +8.1 If any provisions of this Licence are or subsequently become +invalid or unenforceable for any reason, the remaining +provisions shall remain effective. + +8.2 You shall not use any of the name (including acronyms and +abbreviations), image, or logo by which the Licensor or CERN is +known, except where needed to comply with section 3, or where +the use is otherwise allowed by law. Any such permitted use +shall be factual and shall not be made so as to suggest any kind +of endorsement or implication of involvement by the Licensor or +its personnel. + +8.3 CERN may publish updated versions and variants of this Licence +which it considers to be in the spirit of this version, but may +differ in detail to address new problems or concerns. New +versions will be published with a unique version number and a +variant identifier specifying the variant. If the Licensor has +specified that a given variant applies to the Covered Source +without specifying a version, You may treat that Covered Source +as being released under any version of the CERN-OHL with that +variant. If no variant is specified, the Covered Source shall be +treated as being released under CERN-OHL-S. The Licensor may +also specify that the Covered Source is subject to a specific +version of the CERN-OHL or any later version in which case You +may apply this or any later version of CERN-OHL with the same +variant identifier published by CERN. + +8.4 This Licence shall terminate with immediate effect if You fail +to comply with any of its terms and conditions. + +8.5 However, if You cease all breaches of this Licence, then Your +Licence from any Licensor is reinstated unless such Licensor has +terminated this Licence by giving You, while You remain in +breach, a notice specifying the breach and requiring You to cure +it within 30 days, and You have failed to come into compliance +in all material respects by the end of the 30 day period. Should +You repeat the breach after receipt of a cure notice and +subsequent reinstatement, this Licence will terminate +immediately and permanently. Section 6 shall continue to apply +after any termination. + +8.6 This Licence shall not be enforceable except by a Licensor +acting as such, and third party beneficiary rights are +specifically excluded. \ No newline at end of file diff --git a/_sources/FPGA_NETWORK.md.txt b/_sources/FPGA_NETWORK.md.txt new file mode 100644 index 00000000..cad5e8d4 --- /dev/null +++ b/_sources/FPGA_NETWORK.md.txt @@ -0,0 +1,39 @@ +# FPGA network + +The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 is lower port (when PCIe connector is on the bottom). +The card FPGA design is offered in two variants `100g` and `8x10g`. These have different behavior regarding the network: + +`100g` this variant operates QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a **switch**. +QSFP28 transceivers are necessary. + +`8x10g` this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables + are necessary. It is designed for **detector directly connected** to the Jungfraujoch server, without switch. + +## Transceivers +AMD doesn't provide transceiver compatibility matrix for Alveo U55C. +In our experience operating the card we haven't seen issues with transceivers from various providers (FS.com, Mellanox, Finnisar). +We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn't +support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven't +encountered such a situation. + +## Switch configuration +Special care has to be taken for switch operation, given the FPGA core doesn't support auto-negotiation. It is necessary to configure switch port +to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000). + +## Network LEDs +Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). +LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. +For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs means all ports connected, orange LEDs at least one of the ports connected. + +## Network stack +Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. +Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed. + +The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. +The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. +The card won't respond to broadcast ICMP pings. + +Each link can be put in `direct` mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. +This settings should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any +10 Gbit/s link on the same card. Currently `direct` mode is turned OFF for `100g` design and ON for `8x10g` design. +This can be manually adjusted for each link. \ No newline at end of file diff --git a/_sources/FPGA_PCIE_DRIVER.md.txt b/_sources/FPGA_PCIE_DRIVER.md.txt new file mode 100644 index 00000000..743d7f24 --- /dev/null +++ b/_sources/FPGA_PCIE_DRIVER.md.txt @@ -0,0 +1,100 @@ +# FPGA PCIe driver + +## Compilation +To compile kernel module type: +``` +make +``` + +## Installation +To install kernel module, you need to have root permissions and run: +``` +sudo make install +``` + +## Loading driver into kernel +After installing the kernel driver, it should be possible to insert it into the kernel via: +``` +modprobe jfjoch +``` + +## Ownership of the character devices +By default, character devices `/dev/jfjoch` are owned by root (user/group) and are not accessible by others. +This means that `jfjoch_broker` must be running as superuser, which might not be optimal for security reasons in most cases. +The behavior can be changed by creating `udev` rules. Create a file called `/etc/udev/rules.d/99-jfjoch.rules` +with the following content: +``` +KERNEL=="jfjoch*" OWNER="" GROUP="" +``` +It is OK to provide only group, for example to make the devices accessible by group `jungfrau`: +``` +KERNEL=="jfjoch*" GROUP="jungfrau" +``` + +## DKMS +To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel +module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. +For other systems, it is necessary to follow the procedure below, though it is not well tested. + +This first requires to install DKMS - for RHEL it is available via EPEL repository: +``` +sudo dnf install dkms +``` +Then use script provided in the driver directory to copy driver code to DKMS directory: +``` +./install_dkms.sh +``` +If upgrading the driver, please first remove current driver from DKMS system: +``` +dkms remove jfjoch -v --all +``` + +## Driver parameters +Currently, there is one driver parameter `nbuffers`, that defines count of exchange buffers (see below). +This can be adjusted in the modprobe operation, for example: +``` +modprobe jfjoch nbuffers=1024 +``` + +## Exchange buffers +The parameter defines number of buffers used to exchange data between card and host application. +Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. +These buffers are used by both card-to-host and host-to-card operations. + +Buffers use special allocation, as they are continuous in physical address space, which helps the FPGA card to transfer all +data associated with detector module in two DMA transfers (one data, one metadata). +Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in power of two, **4 MiB** is safe number for one buffer size. +Buffer can be mapped into user space, but performing `mmap` system call on the `/dev/jfjoch` character device. + +Buffer count can be adjusted by setting `nbuffers` parameter. There are two considerations for setting optimal value: +1. For card-to-host transfers, minimal value is roughly +` * `, +this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64. +2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. +So minimal count is ` * (3 + 3 * )`. + +Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration. + +## Known problems +To avoid inconsistent behavior, this driver won't load if release number differs between the kernel driver and FPGA card. + +## CMake file +While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. +It is not made for actual compilation of the kernel driver and should not be used for that purpose. + +## Character device access +For each FPGA device a character device is created called `/dev/jfjoch`. +When device is opened two operations are possible: +mmap() to map exchange buffers +ioctl() to communicate with the cards +Interfacing should be done through the JungfraujochDevice class in `fpga/host_library` directory. + +## Sysfs access +Certain performance counters can be read through sysfs mechanism in the kernel. +One needs to `cat` files in `/sys/class/misc/jfjoch/` directory. + +## RHEL 9.5+ issue +RedHat Enterprise Linux 9.5 backported modification to settings virtual memory flags from Linux kernel 6.3, while still operating kernel version 5.14. +It is complicated to come up with a single rule to select when newer functions should be used, so it works with RHEL 9.5+, +while still being compatible with other Linux distributions. It is even more complex given not all RHEL compatible distributions adopted the change at the same version. +For the moment the quick fix is to define an environment variable `HAVE_VM_FLAGS_SET` before making the kernel. diff --git a/_sources/FPGA_SETTINGS.md.txt b/_sources/FPGA_SETTINGS.md.txt new file mode 100644 index 00000000..b5e7bce0 --- /dev/null +++ b/_sources/FPGA_SETTINGS.md.txt @@ -0,0 +1,121 @@ +# FPGA advanced reference +## Register map +FPGA setup can be done via registers: + +| Address | Bits | Meaning | Mode | Notes | +|---------------------|------|------------------------------------------------------------------------------------------------|:-----|----------------------------------------------| +| 0x000000 - 0x00FFFF | | Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory) | | | +| 0x010000 | 32 | Action Control Register | | | +| | | Bit 0 - Action start | R/W | | +| | | Bit 1 - Action idle | R | | +| | | Bit 2 - Action cancel | R/W | cleared on reset or action start | +| | | Bit 3 - Clear network counters | R/W | cleared on reset | +| | | Bit 12:4 - Debug signals (see action_config.v for details) | R | | +| | | Bit 16 - AXI Mailbox interrupt 0 | R | | +| 0x010004 | 32 | Reserved | - | | +| 0x010008 | 32 | Reserved | - | | +| 0x01000C | 32 | GIT SHA1 | R | | +| 0x010010 | 32 | Reserved | R | | +| 0x010014 | 32 | Reserved | R | | +| 0x010018 | 32 | Jungfraujoch FPGA variant | R | | +| 0x01001C | 32 | Reserved | R | | +| 0x010020 | 32 | Max. number supported detector modules | R | constant | +| 0x010024 | 32 | Reserved | R | constant | +| 0x010028 | 64 | Pipeline stalls before writing to host memory | R | reset on action start | +| 0x010030 | 64 | Pipeline stalls before accessing HBM | R | reset on action start | +| 0x010038 | 32 | FIFO status (see action_config.v for details) | R | | +| 0x01003C | 32 | Size of single HBM channel in bytes (default value for the particular card) | R/W | should not be altered for standard operation | +| 0x010040 | 64 | Packets processed by the action | R | cleared on reset or action start | +| 0x010048 | 64 | Valid ethernet packets | R | cleared on reset | +| 0x010050 | 64 | Valid ICMP packets | R | cleared on reset | +| 0x010058 | 64 | Valid UDP packets | R | cleared on reset | +| 0x010060 | 64 | Valid detector packets processed by the card | R | cleared on reset | +| 0x010068 | 64 | Packets flagged as errors by CMAC | R | cleared on reset | +| 0x010070 | 64 | Pipeline stalls before data processing | R | reset on action start | +| 0x010078 | 64 | AXI-beats before accessing HBM | R | reset on action start | +| 0x010080 | 64 | AXI-beats before data processing | R | reset on action start | +| 0x010088 | 64 | AXI-beats before host writer | R | reset on action start | +| 0x010090 | 64 | Last encountered SwissFEL pulse ID | R | cleared on reset | +| 0x010100 | 32 | Spot finder photon count threshold | R/W | | +| 0x010104 | 32 | Spot finder signal-to-noise ratio threshold (single-precision float) | R/W | | +| 0x010200 | 64 | MAC address source for internal frame generator | R/W | network byte order | +| 0x010208 | 32 | IPv4 address source for internal frame generator | R/W | network byte order | +| 0x01020C | 32 | Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.) | R/W | | +| 0x010210 | 32 | Data collection mode | R/W | | +| | | Bit 0 - Conversion to photons | | | +| | | Bit 1 - Output extend to 32-bit | | | +| | | Bit 2 - Output is unsigned integer | | | +| | | Bit 3 - Use sq. root lossy compression | | | +| | | Bit 7 - JUNGFRAU fixed G1 mode | | | +| | | Bit 8 - Set to zero values below threshold | | | +| | | Bit 16:31 - Data collection ID (carried with completions) | | | +| 0x010214 | 32 | Photon energy in keV (single-precision float) | R/W | | +| 0x010218 | 32 | Number of frames expected in the data collection (defines termination condition) | R/W | | +| 0x01021C | 32 | Number of storage cells | R/W | | +| 0x010220 | 32 | Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.) | R/W | | +| 0x010224 | 32 | Coefficient for sq. root compression (need to set bit in data collection mode to apply) | R/W | | +| 0x010225 | 32 | Threshold; set values below set to zero (need to set bit in data collection mode to apply) | R/W | | +| 0x030000 - 0x03FFFF | | AXI Mailbox for Work Request / Work Completion | | See Xilinx PG114 for register map | +| 0x040000 - 0x04FFFF | | QuadSPI flash | | See Xilinx PG153 for register map | +| 0x050000 - 0x05FFFF | | Interrupt controller | | See Xilinx PG099 for register map | +| 0x060000 - 0x06FFFF | | Load calibration (HLS) | | | +| 0x070000 - 0x07FFFF | | AXI Firewall | | See Xilinx PG293 for register map | +| 0x080000 - 0x08FFFF | | Frame generator (HLS) | | | +| 0x090000 - 0x09FFFF | | PCIe DMA control | | See Xilinx PG195 for register map | +| 0x0A0000 - 0x0AFFFF | | I2C clock generator | | See Xilinx PG195 for register map | +| 0x0C0000 - 0x0FFFFF | | Xilinx Card Management Solution Subsystem management subsystem | | See Xilinx PG348 for register map | +| 0x100000 - 0x10FFFF | | MAC 10G / CMAC 100G | | See Xilinx PG210/PG203 for register map | +| 0x110000 - 0x11FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x120000 - 0x12FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x130000 - 0x13FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x140000 - 0x14FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x150000 - 0x15FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x160000 - 0x16FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x170000 - 0x17FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x200000 - 0x20FFFF | | Eth/IPv4 network stack for interface #0 | | | +| 0x210000 - 0x21FFFF | | Eth/IPv4 network stack for interface #1 | | | +| 0x220000 - 0x22FFFF | | Eth/IPv4 network stack for interface #2 | | | +| 0x230000 - 0x23FFFF | | Eth/IPv4 network stack for interface #3 | | | +| 0x240000 - 0x24FFFF | | Eth/IPv4 network stack for interface #4 | | | +| 0x250000 - 0x25FFFF | | Eth/IPv4 network stack for interface #5 | | | +| 0x260000 - 0x26FFFF | | Eth/IPv4 network stack for interface #6 | | | +| 0x270000 - 0x27FFFF | | Eth/IPv4 network stack for interface #7 | | | +| 0x400000 - 0x47FFFF | 64 | Address table: decodes handles used by load_calibration and host_writer to DMA addresses | | | + +## AXI Mailbox + +AXI mailbox is used to send work request from host to action, and receive work completions. +Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114). + +Work request has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------------------------| +| 0 | 15 | Work request ID (handle) | + +Work completion has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------| +| 0 | 15 | Work request ID (handle) | +| | | Special values: | +| | | 65534 - start of data collection | +| | | 65535 - end of data collection | +| 15 | 31 | Data collection ID | + +## HBM memory + +| Interface number | Core | Meaning | +|------------------|------------------|---------------------------------------------| +| 0-1 | jf_conversion | Gain factor G0 | +| 2-3 | jf_conversion | Gain factor G1 | +| 4-5 | jf_conversion | Gain factor G2 | +| 6-7 | jf_conversion | Pedestal G0 | +| 8-9 | jf_conversion | Pedestal G1 | +| 10-11 | jf_conversion | Pedestal G2 | +| 12-13 | integration | Integration map | +| 14-15 | integration | Integration weights | +| 16-17 | spot_finder_mask | Spot finder resolution | +| 18-19 | roi_calc | ROI calculation | +| 20-21 | frame_generator | Frame generator | +| 22-27 | load_from_hbm | Frame summation | diff --git a/_sources/HARDWARE.md.txt b/_sources/HARDWARE.md.txt new file mode 100644 index 00000000..527c2844 --- /dev/null +++ b/_sources/HARDWARE.md.txt @@ -0,0 +1,55 @@ +# Hardware requirements +Operating Jungfraujoch requires the following: + +1. High performance server +2. FPGA board(s) installed in the server +3. (optionally) GPU boards +4. (optionally) 100G switch to connect FPGA and the detector + +Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include +AMD FPGA boards. Therefore, the two has to be purchases separately. This might have impact on the warranty for the hardware +and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn't take any responsibility +for warranty limitations for operating FPGA boards in the server. Having said this - we didn't encounter any hardware issues so far. + +## High performance server +PSI is using HPE DL380 Gen11 servers are the moment to operate Jungfraujoch systems. However, this is because of general +preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system +from any other vendor with similar specification should work as well. + +At PSI, we use the following configuration of HPE DL380 Gen11 to operate 9M pixel detectors at 2 kHz is as follows: +* 2 x Intel Xeon 8558P +* 512 GB RAM +* 2 x Nvidia L4 GPU (for indexing) +* 1 x Nvidia Connect-X 6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs) +* Copper 1G/10G network + +### PCI slots +When ordering the system it is important to ensure enough PCIe cards can be accommodated in the system. +In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2x GPU, 1x network + +Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK. + +## FPGA +Jungfraujoch is built for [AMD/Xilinx U55C](https://www.amd.com/en/products/accelerators/alveo/u55c/a-u55c-p00g-pq-g.html) +(A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported. + +Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. +For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires +using TX delay functionality in the detector. + +## GPUs +Operating fast-feedback indexer code requires operation of a graphic processing unit from Nvidia. +For practical reasons, i.e. power consumption and cost, we choose inference grade card Nvidia L4. +In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work. + +## Network switch +Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs `8x10g` variant +of the Jungfraujoch FPGA image, which allows to directly connect 4 JUNGFRAU modules to one U55C card. + +Such configuration is however +impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujochs server. +In this case one needs a network switch. + +We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. +For switches with only 100G ports it is important to ensure, that these can be split into 4x10G ports to connect the detector. + diff --git a/_sources/HDF5.md.txt b/_sources/HDF5.md.txt new file mode 100644 index 00000000..ce342569 --- /dev/null +++ b/_sources/HDF5.md.txt @@ -0,0 +1,471 @@ +# HDF5 / NeXus data format + +Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html)-compliant. On top of the +NXmx application definition, Jungfraujoch records a substantial amount of *derived* metadata +(spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These +extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and +reusable. + +This page documents the **file layout and the data fields**. The operational behaviour of the +writer (running, republishing, file finalisation, CBF/TIFF output) is described in +[jfjoch_writer](JFJOCH_WRITER.md). The wire format that feeds the writer is described in +[CBOR messages](CBOR.md); fields below frequently correspond one-to-one to CBOR message fields, and +that document is a useful companion for their meaning. + +## 1. Motivation: derived metadata and FAIR data + +The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep +them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both +**rotation** macromolecular crystallography (single- and multi-crystal, including fine-sliced and +helical scans) and **serial** crystallography (stills, grid scans); the same concerns apply to both: + +* **Findability.** Raw diffraction images carry almost no descriptive metadata about *content*. + Quantities such as background level, number of diffraction spots, or indexing outcome let a user + judge the quality and relevance of a dataset *before* inspecting the raw images. +* **Accessibility at scale.** A single experiment can span tens to hundreds of terabytes. Standard + retrieval (e.g. HTTP) makes a dataset *available* but not *inspectable* — users would otherwise + have to download a large fraction of the data just to decide whether it is useful. Compact + derived representations make discovery, assessment and reuse feasible. + +Because Jungfraujoch couples acquisition with real-time analysis used to *steer* experiments, +transparency and reproducibility of that analysis matter. As a minimum the writer therefore +preserves spot-finding and indexing results together with the filters that were applied, and it can +retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse. + +### Two complementary layouts: per-image spots vs. a reflection table + +Jungfraujoch stores analysis products in two shapes, matching how each is accessed. + +**Per-image spot finding / indexing.** Spot finding and indexing are inherently *image-centric* — +the natural query is "give me the spots for image *n*" — and this holds for serial stills and for +rotation frames alike. For these products Jungfraujoch adopts a layout similar to the +[Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the convention +understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties (position, +intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, +with each image allocated room for up to a predefined maximum number of spots. These dense arrays +are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved +without traversing variable-length structures. The cost is some storage overhead for unused slots +(padded with sentinels), which is acceptable for the access pattern. + +**Integrated reflections.** Integrated intensities are naturally a *dataset-wide* table, which is +exactly the model of the NeXus +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class. +This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration +results (see §4.2 below). We deliberately do *not* force spot finding/indexing into a single +experiment-wide table: across the hundreds of thousands of patterns typical of serial — or +fine-sliced rotation — experiments, that would require aggregating the whole experiment before the +spots of one image can be read. We encourage the community to develop standardised NeXus application +definitions for image-centric crystallography products that combine NeXus interoperability with the +access patterns and scale of modern high-throughput experiments. + +## 2. File layout + +A run is written as one **master file** plus, depending on the format, one or more **data files**: + +``` +_master.h5 # NXmx master file (metadata + links / virtual datasets) +_data_000001.h5 # data file: images + per-image analysis +_data_000002.h5 +... +``` + +The master file is produced by `writer/HDF5NXmx.cpp`; data files by `writer/HDF5DataFile.cpp` and +its plugins (`writer/HDF5DataFilePlugin*.cpp`). Files are written to a temporary `*..tmp` +name and renamed on successful close. + +Three master-file variants exist (set via `file_format`): + +| Format | Value | Master ↔ data linking | +|--------|:-----:|------------------------| +| **NXmxLegacy** (default) | 1 | One external link in `/entry/data` per data file (`data_000001`, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0. | +| **NXmxVDS** | 2 | A single virtual dataset `/entry/data/data` spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+. | +| **NXmxIntegrated** | 3 | No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format. | + +In legacy/VDS mode, image-indexed analysis arrays live in the **data files** and are exposed in the +master file through external links or virtual datasets; in integrated mode they are written +directly into the single file. Throughout this document a "✓ in master" column marks entries that +are visible (directly or via link/VDS) from the master file. + +Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or +bitshuffle + Zstd; signed integer image datasets use `INTx_MIN` as the HDF5 fill value (the +"masked / no-data" sentinel), unsigned use `UINTx_MAX`. + +### Reprocessing output: `_process.h5` + +The offline reprocessing tool [`rugnux`](TOOLS.md) (`tools/rugnux_cli.cpp`) re-runs the +full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing +dataset and writes its results to a master file named **`_process.h5`**. This file uses the +**integrated** format, but instead of copying the images its `/entry/data/data` is a *virtual +dataset that links back to the original image files* (`hdf5_source_data` → +`NXmx::LinkToData_ProcessingVDS`). The result is a compact, self-describing companion file that +holds *all* the derived analysis (everything in §4) plus a virtual view +of the raw images — without duplicating terabytes of data. + +This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead +of) the raw data to convey what is in a dataset and how it processed, while the `/entry/data/data` +VDS still resolves to the original images when they are available. `rugnux` can also process +an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. + +## 3. NXmx-standard content + +The entries below are part of, or valid base classes for, the +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html) application definition. +"NXmx" = listed in the application definition; "base" = a valid field of the relevant NeXus base +class (`NXdetector`, `NXsample`, `NXsource`) but not in the NXmx required/recommended subset. + +### `/entry` (NXentry) + +| Field | Std | Notes | +|-------|:---:|-------| +| `definition` | NXmx | value `"NXmx"` | +| `start_time` | NXmx | arming time | +| `end_time`, `end_time_estimated` | NXmx | approximate end time | + +File-level HDF5 attributes `file_name`, `file_time`, `HDF5_Version` are also set. + +### `/entry/source` (NXsource), `/entry/instrument` (NXinstrument) + +| Field | Std | Units | +|-------|:---:|-------| +| `source/name`, `source/type` | NXmx / base | | +| `source/current` | base | A | +| `instrument/name` | NXmx | | + +### `/entry/instrument/beam` (NXbeam) + +| Field | Std | Units | +|-------|:---:|-------| +| `incident_wavelength` | NXmx | angstrom | +| `incident_wavelength_spread` | NXmx | angstrom (only if polychromatic) | +| `total_flux` | NXmx | Hz | + +### `/entry/instrument/attenuator` (NXattenuator) + +| Field | Std | +|-------|:---:| +| `attenuator_transmission` | NXmx | + +### `/entry/instrument/detector` (NXdetector) + +| Field | Std | Units | +|-------|:---:|-------| +| `depends_on` | NXmx | → `transformations/rot3` | +| `beam_center_x`, `beam_center_y` | NXmx | pixel | +| `distance` | NXmx | m | +| `count_time`, `frame_time` | NXmx | s | +| `sensor_thickness` | NXmx | m | +| `sensor_material` | NXmx | | +| `description` | NXmx | | +| `threshold_energy` | NXmx | eV (EIGER; written only for a single channel) | +| `x_pixel_size`, `y_pixel_size` | base | m | +| `serial_number` | base | | +| `bit_depth_readout` | NXmx | | +| `saturation_value` | NXmx | | +| `flatfield_applied` | NXmx | | +| `pixel_mask`, `pixel_mask_applied` | NXmx | `pixel_mask` is `[y, x]`, hard-linked from `detectorSpecific/pixel_mask` | +| `countrate_correction_applied` | NXmx | | +| `number_of_cycles` | base | frame-summation factor | + +### `/entry/instrument/detector/transformations` (NXtransformations) + +The NXtransformations *mechanism* (the `depends_on` chain, `transformation_type`, `vector`, +`offset` attributes) is standard. The axis **names** follow the PyFAI PONI convention chosen by +Jungfraujoch (see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)): + +| Axis | Type | Units | Depends on | +|------|------|-------|-----------| +| `translation` | translation | m | `.` | +| `rot1` | rotation | rad | `translation` | +| `rot2` | rotation | rad | `rot1` | +| `rot3` | rotation | rad | `rot2` | + +The beam centre is encoded in `translation` (its offset from the sample), not only in the +informational `beam_center_x`/`beam_center_y` fields. In a `_process.h5` written by rugnux these axes +carry the **refined** detector geometry — the refined beam centre folds into `translation` and the +refined tilt into `rot1`/`rot2`/`rot3`; the broker writes the user-provided geometry unchanged. + +### `/entry/instrument/detector/module` (NXdetector_module) + +`data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all +NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). + +### `/entry/sample` (NXsample) + +| Field | Std | Units / notes | +|-------|:---:|-------| +| `name` | NXmx | | +| `depends_on` | NXmx | points at the last goniometer / grid-scan axis, or `.` for stills | +| `temperature` | NXmx | K | +| `transformations/` (NXtransformations) | NXmx | rotation axis (e.g. `omega`) or grid-scan translation; hard-linked as `/entry/sample/goniometer` | +| `unit_cell` | base | `[a, b, c, α, β, γ]` | +| `ub_matrix` | base | `[1, 3, 3]`, Angstrom⁻¹ | + +For a rotation scan the goniometer axis is written as a per-image angle array `` plus +`_end`, scalar `_range_average`, `_range_total`, and for helical scans +`_helical_x/_y/_z`. These extra goniometer datasets beyond the bare axis array are Jungfraujoch +conveniences. + +### `/entry/data` (NXdata) + +`data` (3-D image stack, `[n_images, y, x]`) with `image_nr_low` / `image_nr_high` attributes. +In legacy mode this group instead contains one external link `data_000001`, … per data file. + +## 4. Extensions beyond NXmx + +Everything in this section is **outside the NXmx standard**. Each group is declared with +`NX_class = NXcollection` (the NeXus-sanctioned container for non-standardised content) unless noted. +The per-image arrays are indexed by image number, padded to the run length and filled with a +sentinel (`NaN` for floats, `-1`/`0` for integer indices) where a quantity is absent. + +### 4.1 `/entry/MX` — spot finding and indexing (CXI-style) + +The flagship extension. Spot ("peak") properties are stored as fixed-size `[n_images, max_spots]` +arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as `[n_images]` vectors. +In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master. + +**Per-spot arrays `[n_images, max_spots]`:** + +| Dataset | Units | Meaning | Indexing only | +|---------|-------|---------|:---:| +| `peakXPosRaw`, `peakYPosRaw` | pixel | spot position (raw detector frame) | | +| `peakTotalIntensity` | photons | spot intensity | | +| `peakIceRingRes` | | spot lies in an ice-ring resolution band | | +| `peakH`, `peakK`, `peakL` | | Miller indices of the (indexed) spot | ✓ | +| `peakDistEwaldSphere` | Å⁻¹ | distance of the spot from the Ewald sphere | ✓ | +| `peakIndexed` | | spot fits the indexing solution | ✓ | +| `peakLattice` | | lattice the spot belongs to (`-1` = unindexed) | ✓ | + +**Per-image vectors `[n_images]`:** + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `nPeaks` | | number of spots stored for the image (CXI) | +| `strongPixels` | | strong-pixel count (first spot-finding stage) | +| `peakCountUnfiltered` | | spots found before filtering | +| `peakCountLowRes` | | low-resolution spots | +| `peakCountIceRingRes` | | spots inside ice-ring bands | +| `peakCountIndexed` | | spots fitting the indexing solution | +| `imageIndexed` | | image was indexed (0/1) | +| `indexingLatticeCount` | | number of lattices found for the image | +| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A* (2016), Vol. A, [Table 3.1.3.1](https://onlinelibrary.wiley.com/iucr/itc/Ac/ch3o1v0001/table3o1o3o1.pdf)) | +| `bravaisLattice` | | Bravais lattice short code, e.g. `aP`, `mC`, `oF`, `tI`, `hP`, `hR`, `cF` | +| `profileRadius` | Å⁻¹ | crystal profile radius | +| `mosaicity` | deg | mosaicity estimate | +| `bFactor` | Ų | per-image B-factor estimate | +| `resolutionEstimate` | Å | diffraction resolution estimate | +| `integratedReflections` | | number of integrated reflections | +| `bkgEstimate` | photons | mean background in the 3–5 Å resolution band | +| `iceRingScore` | ratio | strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | +| `beam_corr_x`, `beam_corr_y` | pixel | beam-center correction applied during processing | +| `imageScaleFactor` | | on-the-fly per-image scale factor *g* | +| `imageScaleCC` | | on-the-fly scaling correlation coefficient | +| `imageScaleMosaicity` | deg | scaling-model mosaicity | +| `imageScaleBFactor` | Ų | scaling-model B-factor | + +**Per-image lattices:** `latticeIndexed` `[n_images, 9]` (Å) — the real-space lattice (flattened +3×3); `latticeIndexedExtra` `[n_images, max_extra_lattices, 9]` (Å) — additional orientation +variants. + +**Run-level summaries** (written into the master `/entry/MX` at finalisation): + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `indexing_algorithm` | | `FFBIDX` / `FFT (CUDA)` / `FFT (FFTW)` | +| `geom_refinement_algorithm` | | e.g. `beam_center` | +| `rotationLatticeIndexed` | Å | whole-run rotation-indexing lattice (`[9]`) | +| `rotationLatticeIndexedExtra` | Å | additional whole-run lattices (`[m, 9]`) | +| `rotationLatticeNiggliClass` | | Niggli class of the run lattice | +| `imageIndexedMean` | | mean indexing rate over the run | +| `bkgEstimateMean` | photons | mean background over the run | +| `indexedLatticeCount` | | per-image lattice count summary (master). *Note: data files use `indexingLatticeCount`; readers accept either.* | + +CrystFEL can read the spots directly with: + +``` +peak_list = /entry/MX +peak_list_type = cxi +``` + +### 4.2 `/entry/reflections` — integrated reflections (NXreflections) + +Integrated reflections are stored **per image** as +`/entry/reflections/image_NNNNNN` groups, each declared `NX_class = NXreflections`. The columns map +mostly onto the standard +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class: + +| Dataset | Units | NXreflections | Meaning | +|---------|-------|:-------------:|---------| +| `h`, `k`, `l` | | standard | Miller indices | +| `d` | Å | standard | resolution | +| `int_sum` | photons | standard | integrated intensity (summation) | +| `int_err` | photons | non-standard name | σ of the intensity (standard equivalent: `int_sum_errors`) | +| `background_mean` | photons | standard | mean background under the peak | +| `predicted_x`, `predicted_y` | pixel | name standard, units differ | predicted position. NXreflections `predicted_x/_y` are *physical* lengths; the pixel datasets are `predicted_px_x/_y` | +| `observed_x`, `observed_y` | pixel | name standard, units differ | observed centroid (pixels; standard pixel form is `observed_px_x/_y`) | +| `observed_frame` | | standard | image number of the reflection | +| `lp` | | standard | Lorentz–polarization factor (stored as `1/rlp`) | +| `partiality` | | standard | recorded fraction of the reflection | +| `delta_phi` | deg | **extension** | XDS Δφ: offset from the centre of the current frame | +| `zeta` | | **extension** | Lorentz ζ factor (reciprocal-space geometry term) | +| `image_scale_corr` | | **extension** | per-image scale correction; `I_true = image_scale_corr · int_sum` | + +In the master file these per-image groups are exposed through `/entry/reflections` external links +(VDS/integrated formats). + +### 4.3 `/entry/azint` — azimuthal integration + +| Dataset | Shape | Units | Meaning | +|---------|-------|-------|---------| +| `bin_to_q` | `[φ_bins, q_bins]` | Å⁻¹ | q value of each bin | +| `bin_to_two_theta` | `[φ_bins, q_bins]` | deg | 2θ of each bin | +| `bin_to_phi` | `[φ_bins, q_bins]` | deg | azimuthal angle of each bin | +| `image` | `[n_images, φ_bins, q_bins]` | | per-image integrated profile (NaN for empty bins) | +| `image_std` | `[n_images, φ_bins, q_bins]` | | per-bin standard deviation | +| `image_count` | `[n_images, φ_bins, q_bins]` | | pixels contributing per bin | +| `map` | `[y, x]` | | pixel→bin mapping (master file only) | + +### 4.4 `/entry/roi` — regions of interest (per-image results) + +`/entry/roi/` has one sub-group per configured ROI, holding the **per-image result +vectors** `[n_images]`. These are written into the data files; in VDS mode they are exposed from +the master file through virtual datasets, and in integrated mode they are in the single file. +(In legacy mode they remain only in the data files.) + +| Dataset | Meaning | +|---------|---------| +| `max` | maximum pixel value in the ROI | +| `sum` | sum of pixel values | +| `sum_sq` | sum of squared pixel values | +| `npixel` | number of valid pixels | +| `x`, `y` | intensity-weighted centroid | + +### 4.4.1 `/entry/roi_defs` — ROI definitions (master file) + +The **dataset-wide ROI definitions** (geometry, fixed for the whole acquisition) live in the +master file under a *separate* `/entry/roi_defs` group — kept apart from `/entry/roi` above so +that older readers, which iterate `/entry/roi`, are unaffected by these entries. One sub-group +`/entry/roi_defs/` per ROI: + +| Dataset | Meaning | +|---------|---------| +| `bit_index` | which bit of `roi_map` (below) marks this ROI | +| `type` | `box`, `circle` or `azim` | +| `min_x_pxl`, `max_x_pxl`, `min_y_pxl`, `max_y_pxl` | box bounds (type `box`) | +| `center_x_pxl`, `center_y_pxl`, `radius_pxl` | circle (type `circle`) | +| `q_min_recipA`, `q_max_recipA` | Q range (type `azim`) | +| `phi_min_deg`, `phi_max_deg` | azimuthal-angle sector (type `azim`, omitted for a full ring) | + +`/entry/roi_defs/roi_map` `[y, x]` is a `uint16` per-pixel bitmask: bit `bit_index` is set for +every pixel belonging to that ROI, so an ROI's footprint can be recovered exactly. + +### 4.5 `/entry/image` — per-image pixel statistics + +`[n_images]` vectors: `max_value`, `min_value` (viable min/max, excluding error/saturated pixels), +`error_pixels`, `saturated_pixels`, `pixel_sum`. Surfaced in the master file under `/entry/image`. + +### 4.6 `/entry/profiling` — per-image timing + +`[n_images]` vectors in seconds: `spotFindingTime`, `indexingTime`, `integrationTime`, +`refinementTime`, `processingTime`, `braggPredictionTime`, `preprocessingTime`, `compressionTime`, +`azIntTime`, `indexAnalysisTime`, `imageScaleTime`. + +### 4.7 `/entry/detector` — acquisition diagnostics (data file) + +A convenience NXcollection in the data file (note: distinct from the standard +`/entry/instrument/detector`). In **integrated** format these datasets are written under +`/entry/instrument/detector/detectorSpecific` instead. + +| Dataset | Meaning | +|---------|---------| +| `timestamp`, `exptime` | per-image timestamp and exposure time | +| `number` | image number (original number if image rejection was used) | +| `det_info` | JUNGFRAU debug field | +| `storage_cell_image` | storage-cell number | +| `rcv_delay`, `rcv_free_send_buffers` | receiver internal diagnostics | +| `packets_expected`, `packets_received` | UDP packets per image | +| `data_collection_efficiency_image` | received / expected packet ratio | + +### 4.8 `/entry/xfel` — pulsed-source metadata + +`[n_images]` vectors `pulseID` and `eventCode`, written for pulsed sources (e.g. SwissFEL). + +### 4.9 Other collections + +| Path | Class | Content | +|------|-------|---------| +| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode`, `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/`, `data_collection_efficiency_image` | +| `/entry/instrument/detector/calibration` | NXcollection | per-channel pedestal / calibration images (bitshuffle-compressed) | +| `/entry/instrument/fluorescence` | NXcollection | XRF spectrum: `energy` [eV], `data` | +| `/entry/user` | NXcollection | scalar values supplied under `header_appendix.hdf5` | + +### 4.10 Non-standard fields inside the NXmx detector group + +A few extension scalars are written *inside* the otherwise-standard `/entry/instrument/detector` +group for compatibility with existing tooling: + +| Field | Units | Meaning | +|-------|-------|---------| +| `detector_distance` | m | duplicate of `distance` (Dectris/Neggia compatibility) | +| `detector_number` | | detector identifier (Dectris convention) | +| `error_value` | | masked/error pixel sentinel (NXmx standard would be `underload_value`) | +| `bit_depth_image` | | stored image bit depth (NXmx standard is `bit_depth_readout`) | +| `acquisition_type` | | always `triggered` (Dectris convention) | +| `jungfrau_conversion_applied` | | JUNGFRAU photon/keV conversion applied | +| `jungfrau_conversion_factor` | eV | conversion factor | +| `geometry_transformation_applied` | | module→full-detector geometry applied | + +### 4.11 User-supplied metadata: `header_appendix` and `image_appendix` + +Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two +free-form JSON fields in the `/start` request (`broker/jfjoch_api.yaml`) provide this without any +schema change; both accept *any valid JSON*: + +| Field | Carried in | Persisted to HDF5? | +|-------|-----------|--------------------| +| `header_appendix` | the **start** message, under `user_data.user` (see [CBOR](CBOR.md)) | no — except the `hdf5` sub-object (below) | +| `image_appendix` | **every image** message, as `user_data` | no | + +Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, +republished analysis, viewers), so they are the recommended channel for facility- or +beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has +no dedicated API field. + +**Persisting selected values to HDF5.** `header_appendix` is normally *not* written to the master +file. As an exception, if it contains a key `hdf5` whose value is a JSON object of scalars (strings +and numbers — no arrays or nested objects), the writer stores each entry under `/entry/user/`. + +For example, a `/start` request containing: + +```json +{ + "header_appendix": { + "proposal": "p20001", + "operator": "jdoe", + "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 } + }, + "image_appendix": { "trigger_source": "external" } +} +``` + +forwards the whole `header_appendix` as `user_data.user` on the start message and +`{"trigger_source": "external"}` as `user_data` on every image message, and writes three scalars +into the master file: + +``` +/entry/user/beamline = "X06SA" +/entry/user/ring_mode = "top-up" +/entry/user/attenuator_foils = 2 +``` + +## 5. Notes + +* **Units** are written as the HDF5 `units` attribute on the dataset (e.g. `m`, `eV`, `deg`, + `Angstrom`, `Angstrom^-1`, `Angstrom^2`, `pixel`, `s`). +* **Sentinels.** Missing per-image values are `NaN` (floats) or `-1`/`0` (integer indices); image + pixels use `INTx_MIN` / `UINTx_MAX`. +* **Master vs data file.** In legacy/VDS formats the analysis arrays physically live in the data + files; the master file links to them (external links in legacy, virtual datasets in VDS). In the + integrated format there are no data files and everything is in one place. +* **CXI / CrystFEL.** `/entry/MX` follows the CXI peak-list convention; see + [CXI file format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf). diff --git a/_sources/IMAGE_STREAM.md.txt b/_sources/IMAGE_STREAM.md.txt new file mode 100644 index 00000000..38b90b31 --- /dev/null +++ b/_sources/IMAGE_STREAM.md.txt @@ -0,0 +1,235 @@ + +# Data streams + +Jungfraujoch process (`jfjoch_broker`) operates three outputs. +All three can be operated/enabled independently. +These are: +* **Image** - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket) +* **Preview** - images with metadata at a reduced frame rate (PUB socket) +* **Metadata** - only metadata for all the images, bundled into packages (PUB socket) + +## Image stream +Images (with metadata) are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message), [calibration messages](CBOR.md#calibration-message) and [end message](CBOR.md#end-message) with run metadata. + +If `file_prefix` is not provided for a data collection, images won't be sent to image stream (or its HDF5/CBOR replacements). + +### Splitting image stream +Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. +All sockets will forward start and end messages. Only first socket will forward calibration messages and will be marked to write master file. + +### ZeroMQ image stream +This is using PUSH ZeroMQ socket(s). +It should be strictly avoided to have multiple receivers connected to one PUSH ZeroMQ socket. +ZeroMQ will send the images in a round-robin basis to the receivers. +In this case start and end messages will end up only with one receiver. +Instead, Jungfraujoch feature of multiple sockets should be used. +For ZeroMQ image stream, each writer connects to a different port. + +Behavior is as following: +* Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer. +* Calibration message is sent to the first socket only, with timeout of 1s. +* Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped). +* End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded. + +The format is generally interchangeable with DECTRIS Stream2 format. + +#### ZeroMQ configuration + +ZeroMQ image stream is configured in the broker JSON configuration file under the `zeromq_settings` section: +```json +{ + "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"], + "send_watermark": 100, + "send_buffer_size": 67108864, + "writer_notification_socket": "tcp://192.168.0.1:*" +} +``` + +- `image_socket`: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (`tcp://`, `ipc://`). `0.0.0.0` binds on all network interfaces. +- `send_watermark` (optional): ZeroMQ send high-water mark (number of outstanding messages per socket). +- `send_buffer_size` (optional): OS-level send buffer size for the ZeroMQ socket. +- `writer_notification_socket` (optional): see [Writer notification socket](#writer-notification-socket) below. + +### TCP/IP image stream +This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. +This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication +between the data collection and the writer, and is therefore more robust than ZeroMQ. + +For TCP/IP image stream, Jungfraujoch **listens** on a single TCP port and all writers **connect** to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic `KEEPALIVE` frames when no data collection is active to detect dead connections; writers are expected to respond with a `KEEPALIVE` pong. + +Using `*` as port number (e.g. `tcp://127.0.0.1:*`) is supported — the OS assigns a free port and the actual bound address can be queried via `GetAddress()`. + +Payloads for `START`, `DATA`, `CALIBRATION` and `END` frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages. +`ACK`, `CANCEL`, and `KEEPALIVE` are control frames (no CBOR payload). + +The data collection lifecycle on each connection follows: +`START` → `CALIBRATION` (socket 0 only) → `DATA` (repeated) → `END` + +If a `START` ACK fails on any connection, Jungfraujoch sends `CANCEL` to all already-started connections and rolls back. + +For each frame: +1. Read one `TcpFrameHeader` (fixed size, 64-byte aligned). +2. Validate `magic` (`0x4A464A54` / `"JFJT"`) and `version` (`2`). +3. Read `payload_size` bytes (if non-zero). + +When image stream is split into multiple connections: +- `START` and `END` are sent on all connections, +- `CALIBRATION` is sent only on connection 0, +- `DATA` frames are distributed by file grouping: connection index = `(image_number / images_per_file) % num_connections`. + +#### TCP/IP configuration + +TCP/IP image stream is configured in the broker JSON configuration file under the `tcp_settings` section: +```json +{ + "image_socket": "tcp://192.168.0.1:9100", + "nwriters": 2, + "send_buffer_size": 67108864 +} +``` + +- `addr`: listen address in `tcp://:` format. `0.0.0.0` binds on all interfaces. `*` as port selects a random free port. +- `nwriters` (optional): maximum number of simultaneous writer connections accepted. +- `send_buffer_size` (optional): OS-level `SO_SNDBUF` size for accepted connections. + +#### ACK handling + +ACK handling is mandatory for correct operation: +- `START` **must** be acknowledged (`ACK` with `ack_for=START`) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered. +- `END` **must** be acknowledged (`ack_for=END`) on each connection within 10 seconds for successful completion. +- `CANCEL` should be acknowledged during rollback paths (500ms timeout). +- `DATA` should be acknowledged for every frame. A `DATA` ACK with `FATAL` flag set reports a downstream error (e.g. disk full) which is propagated to `jfjoch_broker` via `Finalize()`. A failed `DATA` ACK does **not** break the TCP connection on its own — data continues to flow. +- `CALIBRATION` is not acknowledged at this time. +- `KEEPALIVE` frames are not acknowledged via ACK; the writer responds with a `KEEPALIVE` pong frame instead. + +#### Keepalive + +When no data collection is active, Jungfraujoch sends `KEEPALIVE` frames approximately every 5 seconds on each persistent connection. Writers should respond with a `KEEPALIVE` frame (pong). OS-level TCP keepalive is also enabled (`TCP_KEEPIDLE=30s`, `TCP_KEEPINTVL=10s`, `TCP_KEEPCNT=3`) as a secondary safety net. Dead connections are automatically removed from the pool. + +#### Zero-copy transmission + +On Linux, large payload transmission (`DATA` and `CALIBRATION` frames) can use kernel TCP zero-copy (`SO_ZEROCOPY`/`MSG_ZEROCOPY`) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal `send()` behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread. + +#### Frame types + +| Value | Name | Purpose | +|---:|---|---| +| 1 | `START` | Start-of-run metadata | +| 2 | `DATA` | One image payload | +| 3 | `CALIBRATION` | Calibration payload | +| 4 | `END` | End-of-run metadata | +| 5 | `ACK` | Acknowledgement / error reporting | +| 6 | `CANCEL` | Cancel run initialization/stream | +| 7 | `KEEPALIVE` | Connection liveness probe/pong | + +#### TCP frame header (`TcpFrameHeader`) + +| Field | Type | Description | +|--------------------------|---|----------------------------------------------------------| +| `magic` | `uint32_t` | Protocol magic (`0x4A464A54`, `"JFJT"`) | +| `version` | `uint16_t` | Protocol version (`2`) | +| `type` | `uint16_t` | Frame type (see table above) | +| `image_number` | `uint64_t` | Image index for `DATA` frames | +| `payload_size` | `uint64_t` | Number of payload bytes after header | +| `socket_number` | `uint32_t` | Connection index in split-stream mode | +| `flags` | `uint32_t` | ACK flags (`OK`, `FATAL`, `HAS_ERROR_TEXT`) | +| `run_number` | `uint64_t` | Run identifier | +| `ack_processed_images` | `uint32_t` | In `ACK`: number of images processed by receiver | +| `ack_code` | `uint16_t` | In `ACK`: error/status code | +| `ack_for` | `uint16_t` | In `ACK`: frame type being acknowledged | +| `ack_fifo_occupancy` | `uint16_t` | In `ACK`: occupancy of input FIFO in the `jfjoch_writer` | +| `ack_fifo_max_occupancy` | `uint64_t` | In `ACK`: max occupancy of input FIFO | + +The header is 64-byte aligned (`alignas(64)`). + +#### ACK semantics + +- `ACK` frames use `ack_for` to indicate which frame type is acknowledged. +- `flags`: + - `OK` (bit 0): operation accepted/successful, + - `FATAL` (bit 1): receiver reports unrecoverable error (primarily for `DATA`), + - `HAS_ERROR_TEXT` (bit 2): ACK payload contains UTF-8 error text. +- `ack_code` can be used to categorize errors: + +| Code | Name | Meaning | +|---:|---|---| +| 0 | `None` | No error | +| 1 | `StartFailed` | START processing failed | +| 2 | `DataWriteFailed` | Image write failed | +| 3 | `EndFailed` | END processing failed | +| 4 | `DiskQuotaExceeded` | Disk quota exceeded | +| 5 | `NoSpaceLeft` | No space left on device | +| 6 | `PermissionDenied` | Permission denied | +| 7 | `IoError` | General I/O error | +| 8 | `ProtocolError` | Protocol-level error | + +### Image stream replacement +Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting "None" image pusher for all the measurements. + +## Writer notification socket +The writer notification socket is used **only with ZeroMQ image stream**. Since ZeroMQ is asynchronous, `jfjoch_broker` does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back. + +For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame. + +To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with `writer_notification_socket` entry: +```json +{ + "writer_notification_socket":"tcp://192.168.0.1:*" +} +``` +Such entry will create PULL socket on `192.168.0.1` network interface listening on one, random TCP port. When data processing is started, the +image stream will send CBOR [start message](CBOR.md#start-message). This message will include information on `writer_notification_zmq_addr`, +which needs to be used by downstream code. Since the start message must reference the address of `jfjoch_broker` host, notification +socket should always listen on a particular network interface, and should not be configured with placeholder address `0.0.0.0`. It is, however, OK +to use placeholder `:*` for network port, as it will be substituted for the one chosen by ZeroMQ. + +For every image stream socket, downstream code must send the following message to the PULL socket: +```json +{ + "run_number":135, + "run_name": "lysozyme_1", + "socket_number": 1, + "processed_images":250, + "ok": true +} +``` +Here `run_number`, `run_name` and `socket_number` must match information from the start message. +`ok` is boolean confirming if the writing process was OK. +`processed_images` is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. +If not, it is possible to include error message: +```json +{ + "run_number":135, + "run_name": "lysozyme_1", + "socket_number": 1, + "processed_images": 0, + "ok": false, + "error": "Permission error" +} +``` +This way errors from the downstream code are propagated to `jfjoch_broker`. + +If writer notification socket is configured, but downstream code doesn't send proper notification, `jfjoch_broker` will time out after 60 seconds producing an error message. + +## Preview stream +Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. +Images are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. +Only start and image messages are sent. + +This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope +with the messages, it will always receive the last generated message (no backlog). +For this reason it is also recommended to use the same option on receiver side. + +Given PUB socket properties, it is possible to connect multiple viewers to a single socket --- all the viewers should receive all the images sent. + +## Metadata stream +Jungfraujoch can also send pure metadata for the purpose of archiving such information. +Metadata are serialized as CBOR [metadata message](CBOR.md#metadata-message). +This is very similar as image message, but excludes the actual image array and spot positions. +As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. +Order of images within bundle, as well a size of the bundle, are not guaranteed. +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. + +This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected. \ No newline at end of file diff --git a/_sources/JFJOCH_BROKER.md.txt b/_sources/JFJOCH_BROKER.md.txt new file mode 100644 index 00000000..76e3f2b7 --- /dev/null +++ b/_sources/JFJOCH_BROKER.md.txt @@ -0,0 +1,188 @@ +# jfjoch_broker + +`jfjoch_broker` is the main service for the Jungfraujoch application. It is responsible for: + +* Providing user interface via HTTP and OpenAPI +* Configuring FPGA firmware +* Building images from FPGA output and forwarding the results over ZeroMQ + +## External interfaces +Broker operates four external interfaces. + +**Image stream** ZeroMQ PULL socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream +processing. See details [here](IMAGE_STREAM.md#image-stream). + +**Preview stream** ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details [here](IMAGE_STREAM.md#preview-stream). + +**Metadata stream** ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details [here](IMAGE_STREAM.md#metadata-stream). + +**Configuration, status and results interface** HTTP/REST interface described in the OpenAPI format. +Description of the API is presented in the [OpenAPI description](../broker/redoc-static.html). + +## Broker configuration +`jfjoch_broker` requires JSON configuration files. The file is described by OpenAPI structure `jfjoch_settings` defined in `jfjoch_api.yaml` file. +It is recommended to go through example files in the `etc/`. + +Example with all fields: + +```json +{ + "pcie": [ + { + "blk": "/dev/jfjoch0", + "ipv4": "10.1.1.7" + }, + { + "blk": "/dev/jfjoch1", + "ipv4": "10.1.1.8" + } + ], + "zeromq": { + "send_watermark": 100, + "send_buffer_size": 1024, + "image_socket": [ + "tcp://1.2.3.4:5000", + "tcp://1.2.3.4:5001" + ], + "writer_notification_socket": "tcp://1.3.4.6:7000" + }, + "instrument": { + "source_name": "Swiss Light Source", + "source_type": "Synchrotron X-ray Source", + "instrument_name": "X06SA", + "pulsed_source": false, + "electron_source": false + }, + "detector": [ + { + "description": "EIGER 1M", + "serial_number": "E1M-01", + "type": "EIGER", + "high_voltage_V": 150, + "udp_interface_count": 1, + "module_sync": true, + "sensor_thickness_um": 320, + "calibration_file": [ + "gainMaps.bin" + ], + "hostname": [ + "e1m-01", + "e1m-02" + ], + "readout_time_us": 3, + "sensor_material": "Si", + "tx_delay": [ + 0,1 + ], + "base_data_ipv4_address": "10.10.10.50", + "standard_geometry": { + "nmodules": 1, + "gap_x": 8, + "gap_y": 36, + "modules_in_row": 1 + }, + "custom_geometry": [ + { + "x0": 0, + "y0": 0, + "fast_axis": "Xp", + "slow_axis": "Xp" + } + ], + "mirror_y": true + } + ], + "detector_settings": { + "frame_time_us": 450, + "count_time_us": 0, + "internal_frame_generator": false, + "internal_frame_generator_images": 1, + "detector_trigger_delay_ns": 0, + "timing": "auto", + "eiger_threshold_keV": 6.0, + "jungfrau_pedestal_g0_frames": 2000, + "jungfrau_pedestal_g1_frames": 300, + "jungfrau_pedestal_g2_frames": 300, + "jungfrau_pedestal_g0_rms_limit": 100, + "jungfrau_pedestal_min_image_count": 128, + "jungfrau_storage_cell_count": 1, + "jungfrau_storage_cell_delay_ns": 5000, + "jungfrau_fixed_gain_g1": false, + "jungfrau_use_gain_hg0": false + }, + "azim_int": { + "polarization_factor": -1, + "solid_angle_corr": true, + "high_q_recipA": 0, + "low_q_recipA": 0, + "q_spacing": 0 + }, + "image_format": { + "summation": true, + "geometry_transform": true, + "jungfrau_conversion": true, + "jungfrau_conversion_factor_keV": 0.001, + "bit_depth_image": 16, + "signed_output": true, + "mask_module_edges": true, + "mask_chip_edges": true + }, + "image_buffer_MiB": 2048, + "receiver_threads": 64, + "frontend_directory": "/usr/share/jfjoch/frontend", + "image_pusher": "ZeroMQ", + "zeromq_metadata": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4357" + }, + "zeromq_preview": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4356" + } +} +``` + +## Setting up a local test for Jungfraujoch +For development, it is possible to set up a local installation of Jungfraujoch. +This will work without FPGA installed in the computer and allows to test Jungfraujoch software layer, including +ZeroMQ streaming and file writing. + +The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore +very low, as fixed-point calculations have large performance penalty on CPU. In the CPU simulation mode, one can simulate +using only a single FPGA device. + +To run the test: + +### Compile Jungfraujoch with frontend +``` +mkdir build +cd build +cmake .. +make jfjoch +make frontend +``` +Alternatively, for RHEL8 system, you can use RPM generated by automated pipeline. +Solely `jfjoch` one is enough. +In this case - it is necessary to update `etc/broker_local.json` file with frontend path in `/usr/share/jfjoch/frontend`. + +### Start service +Start broker: +``` +cd build/broker +./jfjoch_broker ../../etc/broker_local.json 5232 +``` + +### Run tests +To run test a Python script is provided: +``` +cd tests/test_data +python jfjoch_broker_test.py +``` +The script will initialize Jungfraujoch, import test image and start data collection. + +### Expected result +You can observe online data analysis by opening the following web page: [http://localhost:5232](http://localhost:5232). +Also, a dataset with images should be written in the `build/broker` directory. + diff --git a/_sources/JFJOCH_VIEWER.md.txt b/_sources/JFJOCH_VIEWER.md.txt new file mode 100644 index 00000000..c36b4464 --- /dev/null +++ b/_sources/JFJOCH_VIEWER.md.txt @@ -0,0 +1,133 @@ +# jfjoch_viewer + +`jfjoch_viewer` is the **interactive** desktop application of Jungfraujoch. It opens diffraction +datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal +integration, per-image statistics), and can follow a live data collection by syncing with a +running [`jfjoch_broker`](JFJOCH_BROKER.md) over its HTTP interface. + +It is a standalone Qt 6 application, distributed pre-built on the Gitea release page and in the +Jungfraujoch RPM/APT repositories (see [Deployment](DEPLOYMENT.md)). + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| **`jfjoch_viewer`** | **Interactive, on-screen exploration** | **Qt desktop application** | **Displayed on screen (results not saved to disk)** | +| [`rugnux`](RUGNUX.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | + +## Functionality + +- Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the + `*_process.h5` files produced by [`rugnux`](RUGNUX.md). It also opens NXmx files + written by DECTRIS detectors, though that path has had only limited testing. +- Runs an **embedded data-processing pipeline** — the same analysis code as the rest of + Jungfraujoch — performing spot finding, indexing and integration on the displayed images. + Results are shown on screen but are **not** saved to disk. +- Auxiliary windows and panels: image list, image metadata, spot list, reflection list, + per-region-of-interest statistics, the azimuthal-integration profile, and dataset-info charts. +- User-mask editing: build a user mask interactively, clear it, save it as TIFF, or upload it to a + connected server. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. On systems with a +GPU, use the CUDA build (provided as separate RPM/APT repositories) for the embedded indexing and +integration; the non-CUDA build runs the same pipeline on the CPU at much lower throughput. + +## Opening data + +- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file. +- **File ▸ Open HTTP** (`Ctrl+H`) — connect to a `jfjoch_broker` HTTP endpoint to follow a live + collection. The dialog defaults to host `localhost` and port `8080`; these defaults can be + overridden with the environment variables `JUNGFRAUJOCH_HTTP_HOST` and `JUNGFRAUJOCH_HTTP_PORT`. +- **Command line** — `jfjoch_viewer ` opens a file (or an `http://host:port` URL) on + start-up. `--dbus ` (`-d`) enables or disables the D-Bus interface (default: enabled); + `--help` and `--version` behave as usual. + +## D-Bus interface + +When enabled, the viewer registers the D-Bus interface `ch.psi.jfjoch_viewer`, so other processes +can drive it: + +- `LoadFile(filename, image_number=0, summation=1)` — open a file (or an `http://host:port` URL) + and display the given image. +- `LoadImage(image_number, summation=1)` — navigate to an image in the already-open dataset. + +`summation` sums that many consecutive images before display. + +## Building from source on Windows + +`jfjoch_viewer` is the one Jungfraujoch component that is cross-platform: it builds on Windows 11 +with MSVC and the full CUDA GPU path. (The rest of Jungfraujoch — broker, receiver, FPGA host — is +Linux-only.) There is no pre-built Windows package yet, so build it from source. On Windows the +build is automatically restricted to the viewer and the libraries it needs (`JFJOCH_VIEWER_ONLY` is +forced on), and the remaining dependencies are fetched and built automatically (the first configure +needs network access). + +Verified toolchain: + +- Windows 11 +- Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC +- CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path +- Qt 6.11 for MSVC (`msvc2022_64`), including the **Qt Charts** module — e.g. `C:\Qt\6.11.1\msvc2022_64` +- CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of + the box — it comes with the C++ workload, so there is nothing extra to install. Any recent + standalone CMake (from cmake.org, or the one bundled with Qt in `C:\Qt\Tools\CMake_64`) works too. +- zlib and Eigen — the two libraries not auto-fetched on Windows. Build/install both into one prefix + (here `C:\deps`) and point CMake at it: + ``` + :: static zlib + git clone --branch v1.3.1 https://github.com/madler/zlib + cmake -G Ninja -S zlib -B zlib-build -DCMAKE_INSTALL_PREFIX=C:/deps + cmake --build zlib-build --target install + :: Eigen 3.4 (header-only) -- install just the headers with `cmake --install`; the BLAS/LAPACK/test + :: targets are disabled since they are not needed (and fail to build under MSVC). Use the 3.4 series: + :: the project requests find_package(Eigen3 3.4), which Eigen's same-major rule rejects for 5.x. + git clone --branch 3.4.0 https://gitlab.com/libeigen/eigen.git + cmake -G Ninja -S eigen -B eigen-build -DCMAKE_INSTALL_PREFIX=C:/deps ^ + -DEIGEN_BUILD_BLAS=OFF -DEIGEN_BUILD_LAPACK=OFF -DEIGEN_BUILD_DOC=OFF -DBUILD_TESTING=OFF + cmake --install eigen-build + ``` +- Optional: [NSIS](https://nsis.sourceforge.io) to build the `.exe` installer. + +Configure and build from an **x64 Native Tools Command Prompt for VS 2026** (so `cl`, `nvcc` and +`ninja` are on `PATH`): + +``` +cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_PREFIX_PATH="C:/deps;C:/Qt/6.11.1/msvc2022_64" +cmake --build build-win --target jfjoch_viewer +``` + +Notes: + +- `CMAKE_PREFIX_PATH` (the `C:/deps` prefix plus Qt) is the only required flag — CMake finds zlib and + Eigen from the prefix, so no separate `-DZLIB_ROOT` is needed. +- The CUDA toolchain is located automatically from the `CUDA_PATH` environment variable that the + CUDA installer sets (or from `nvcc` on `PATH`). Pass `-DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe"` + only if `nvcc` is installed in a nonstandard location and is not found. +- For a machine without an NVIDIA GPU, add `-DJFJOCH_USE_CUDA=OFF`: the viewer then runs the same + pipeline on the CPU (FFTW indexer) at lower throughput. + +To produce a self-contained installer (bundles the Qt runtime via `windeployqt`, the analysis CLIs, +and — on the CUDA build — the cuFFT runtime DLL, so the target host needs no Qt and no CUDA toolkit, +only an NVIDIA GPU driver), with NSIS installed: + +``` +cd build-win +cpack +``` + +The NSIS generator is selected automatically on Windows (no `-G` needed). The installer filename and +the Add/Remove Programs entry mark the CUDA/CPU variant, while the install folder and Start Menu +group stay plain `Jungfraujoch` (the two variants share an install location and replace each other — +CUDA is a strict superset): + +| Build | Installer file | Add/Remove Programs | +| --- | --- | --- | +| CUDA (default) | `jfjoch--win64-cuda.exe` | `Jungfraujoch (CUDA)` | +| `-DJFJOCH_USE_CUDA=OFF` | `jfjoch--win64-cpu.exe` | `Jungfraujoch (CPU)` | + +`` is the CUDA toolkit major version (e.g. `cuda13`). The cuFFT DLL is ~256 MB, so the CUDA +installer is correspondingly larger — hence the variant tag in the filename. diff --git a/_sources/JFJOCH_WRITER.md.txt b/_sources/JFJOCH_WRITER.md.txt new file mode 100644 index 00000000..40b094e8 --- /dev/null +++ b/_sources/JFJOCH_WRITER.md.txt @@ -0,0 +1,166 @@ +# jfjoch_writer + +`jfjoch_writer` is NeXus compliant HDF5 file writer. + +## Acknowledgements +* Zdenek Matej (MAX IV) +* Felix Engelmann (MAX IV) +for testing and multiple improvement suggestions. + +## Running directory +Writer needs to be running in base directory for writing files - `file_prefix` will be always relative in regard to writer running directory. +Writer detects and protects for basic security issues, like `file_prefix` starting with a slash, or starting with `../`, or containing `/../`. + +## Usage +Writer needs to be started as a background service, with the following command: +``` +jfjoch_writer {options}
+ +Options: +-R | --root_dir= Root directory for file writing +-H | --http_port= HTTP port for statistics +-r | --zmq_repub_port= ZeroMQ port for PUSH socket to republish images +-f | --zmq_file_port= ZeroMQ port for PUB socket for notifications on finalized files +-w | --rcv_watermark= Receiving ZeroMQ socket watermark (default = 100) +-W | --repub_watermark= Republish ZeroMQ socket watermark (default = 1000) +``` +for example: +``` +jfjoch_writer -H5234 tcp://dcu-address:5400 +``` + +## HTTP interface +Writer has dedicated status interface via HTTP. It allows for two operations: +* ***check state of the writer*** to check if the writer is properly synchronized with DCU (e.g., that `file_prefix` agrees with what was set on the DCU) and monitor progress. +* ***cancel writing*** this will close all the HDF5 files being written and restart writer - the option should be used only if DCU process was terminated or disconnected, it SHOULD NOT be used as standard cancellation procedure (when DCU received cancel command it should properly finish writing as well) + +## Republish +Republish creates a PULL socket on the writer, where all the messages are republished for further use by data analysis pipeline. +Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won't be republished. +In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won't be retried. + +Republish functionality is optional, if republish port number is omitted this functionality is not enabled. + +## Overwriting files +When `jfjoch_writer` creates a HDF5 file, it first adds suffix `..tmp`. +Random value depends on current time-stamp and likely will be different from each file of the particular series. +After file is all saved and closed, it is renamed to remove the suffix. +By default, renaming won't happen if this would overwrite existing file. +However, this behavior can be changed by setting `overwrite` parameter to true in the file writer configuration. + +### When the overwrite conflict is reported +An existing output file is a fatal condition (unless `overwrite` is true). *When* it is detected +depends on whether the transport between the broker and the writer has a back-channel to report the +failure before acquisition starts: + +* **Direct HDF5 pusher and TCP writer (back-channel available).** The conflict is detected at + **start**: the writer that owns the master file checks whether it already exists and refuses to + start. The direct pusher raises the error in-process; the TCP writer returns a START-failure + acknowledgement. Either way the broker learns immediately and aborts the data collection *before* + the detector is armed — no images are taken and nothing is written. Only the master file is + checked up front: in a multi-writer setup the per-image data files are staggered across writers, + and checking them at start would make each writer inspect files it never writes (and race the + writers that do). Data-file conflicts are instead caught by their owning writer at the final + rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker. +* **ZeroMQ writer (no back-channel).** The ZeroMQ image stream is fire-and-forget: the writer has no + way to tell the broker to stop, and the broker would keep streaming images regardless. The writer + therefore does **not** fail at start. It writes the whole series to the `..tmp` files as + usual and only fails at the final rename, leaving the `.tmp` files on disk. This is deliberate: the + acquired images are preserved (in `.tmp` form) rather than being dropped by a writer that aborted + mid-stream. Rename the `.tmp` files by hand, or re-run with `overwrite` set, to recover them. + +## Finalized files information +Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure: + +``` +{ + "filename": : HDF5 data file name (relative to writer root directory), + "nimages": number of images in the file (counting from 1!), + "file_number": number of file within the acquisition, + "sample_name": name of sample, + "run_name": name of run, + "run_number": number of run, + "experiment_group": number of p-group / proposal (optional), + "user_data": user_data, + "beam_x_pxl": beam center (X) in pixels, + "beam_y_pxl": beam center (Y) in pixels, + "detector_distance_m": detector distance (X) in m, + "detector_height_pxl": detector size (X) in pixels, + "detector_width_pxl": detector size (Y) in pixels, + "incident_energy_eV": photon energy of the X-ray beam, + "pixel_size_m": pixel size in meter (assuming pixel X == Y), + "saturation": this count and higher mean saturation, + "space_group_number": space group number (optional), + "underload": pixels with this count should be excluded, + "unit_cell": unit cell dimensions in Angstrom/degree { + "a": , "b": , "c": , + "alpha": , "beta": , "gamma": + }, +} +``` +`user_data` is defined as `header_appendix` in the `/start` operation in the `jfjoch_broker`. +Other metadata are also carried over from `/start` operation. + +If the `header_appendix` is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. +For example `header_appendix` of `{"param1": "test1", "param2": ["test1", "test2"]}`, than example message will look as follows: +```json +{ + "filename": "dataset_name_data_000001.h5", + "nimages": 1000, + "file_number": 0, + "sample_name": "lysozyme", + "run_name": "lyso_cryo", + "run_number": 25, + "experiment_group": "p00001", + "beam_x_pxl": 1200, + "beam_y_pxl": 1500, + "detector_distance_m": 0.155, + "detector_height_pxl": 2164, + "detector_width_pxl": 2068, + "image_time_s": 0.001, + "nimages": 2, + "incident_energy_eV": 12400.0, + "pixel_size_m": 7.5e-05, + "saturation": 32766, + "space_group_number": 96, + "underload": -32768, + "unit_cell": { + "a": 78.0, + "alpha": 90.0, + "b": 78.0, + "beta": 90.0, + "c": 39.0, + "gamma": 90.0 + }, + "user_data": { + "param1": "test1", + "param2": ["test1", "test2"] + } +} +``` + +Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled. + +## HDF5 file structure + +Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, +integration, azimuthal integration, per-image statistics and timing) stored *beyond* the NXmx +standard. The complete file layout — master vs data files, the three format variants +(`NXmxLegacy`, `NXmxVDS`, `NXmxIntegrated`), every NXmx field that is populated and every +Jungfraujoch extension — is documented in [HDF5 / NeXus data format](HDF5.md). + +If data collection was configured with a `header_appendix` containing a key `hdf5` whose value is a +JSON object of numbers and strings, those entries are written to `/entry/user`. + +## Other formats (CBF and TIFF) +In addition to HDF5 format, Jungfraujoch allows to save images in the Crystallographic Binary File (CBF) format. +CBF files are written according to miniCBF format, with only basic header, and always with 32-bit signed integer format. +Dynamic range is reduced to max 2^24, negative numbers are zeroed, and masked, and/or bad pixels are set to -1. + +Also writing to TIFF files is possible, though no metadata are saved in this case. + +## No file option(s) +There are two options to disable writing of files by the writer: +* Setting `file_prefix` to empty string - this will disable sending files on ZeroMQ image socket. +* Setting file format to `NoFile` - files are streamed over ZeroMQ socket, but `jfjoch_writer` will not write anything. +This can be useful for debugging purposes, or if you only rely on republishing functionality of the `jfjoch_writer` \ No newline at end of file diff --git a/_sources/LICENSE.md.txt b/_sources/LICENSE.md.txt new file mode 100644 index 00000000..222b84ba --- /dev/null +++ b/_sources/LICENSE.md.txt @@ -0,0 +1,690 @@ +# License + +Jungfraujoch software is licensed with GPLv3 license. +Jungfraujoch FPGA is licensed with CERN OHL-S license (see [FPGA license](FPGA_LICENSE.md)). + +## GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +### Preamble + +The GNU General Public License is a free, copyleft license for +software and other kinds of works. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + +Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + +Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and +modification follow. + +### TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +1. Source Code. + +The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + +The Corresponding Source for a work in source code form is that +same work. + +2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + +4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + +a) The work must carry prominent notices stating that you modified +it, and giving a relevant date. + +b) The work must carry prominent notices stating that it is +released under this License and any conditions added under section +7. This requirement modifies the requirement in section 4 to +"keep intact all notices". + +c) You must license the entire work, as a whole, under this +License to anyone who comes into possession of a copy. This +License will therefore apply, along with any applicable section 7 +additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no +permission to license the work in any other way, but it does not +invalidate such permission if you have separately received it. + +d) If the work has interactive user interfaces, each must display +Appropriate Legal Notices; however, if the Program has interactive +interfaces that do not display Appropriate Legal Notices, your +work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + +a) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by the +Corresponding Source fixed on a durable physical medium +customarily used for software interchange. + +b) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by a +written offer, valid for at least three years and valid for as +long as you offer spare parts or customer support for that product +model, to give anyone who possesses the object code either (1) a +copy of the Corresponding Source for all the software in the +product that is covered by this License, on a durable physical +medium customarily used for software interchange, for a price no +more than your reasonable cost of physically performing this +conveying of source, or (2) access to copy the +Corresponding Source from a network server at no charge. + +c) Convey individual copies of the object code with a copy of the +written offer to provide the Corresponding Source. This +alternative is allowed only occasionally and noncommercially, and +only if you received the object code with such an offer, in accord +with subsection 6b. + +d) Convey the object code by offering access from a designated +place (gratis or for a charge), and offer equivalent access to the +Corresponding Source in the same way through the same place at no +further charge. You need not require recipients to copy the +Corresponding Source along with the object code. If the place to +copy the object code is a network server, the Corresponding Source +may be on a different server (operated by you or a third party) +that supports equivalent copying facilities, provided you maintain +clear directions next to the object code saying where to find the +Corresponding Source. Regardless of what server hosts the +Corresponding Source, you remain obligated to ensure that it is +available for as long as needed to satisfy these requirements. + +e) Convey the object code using peer-to-peer transmission, provided +you inform other peers where the object code and Corresponding +Source of the work are being offered to the general public at no +charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + +a) Disclaiming warranty or limiting liability differently from the +terms of sections 15 and 16 of this License; or + +b) Requiring preservation of specified reasonable legal notices or +author attributions in that material or in the Appropriate Legal +Notices displayed by works containing it; or + +c) Prohibiting misrepresentation of the origin of that material, or +requiring that modified versions of such material be marked in +reasonable ways as different from the original version; or + +d) Limiting the use for publicity purposes of names of licensors or +authors of the material; or + +e) Declining to grant rights under trademark law for use of some +trade names, trademarks, or service marks; or + +f) Requiring indemnification of licensors and authors of that +material by anyone who conveys the material (or modified versions of +it) with contractual assumptions of liability to the recipient, for +any liability that these contractual assumptions directly impose on +those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + +13. Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +### How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + +You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + +The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + + +## Jungfraujoch exceptions to GPL + +As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL. + +We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose +of simulating FPGA design on CPU. + +If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch +API it may be distributed under terms of your choosing without being subject to GPL requirements. diff --git a/_sources/NAMING.md.txt b/_sources/NAMING.md.txt new file mode 100644 index 00000000..fa6a7a83 --- /dev/null +++ b/_sources/NAMING.md.txt @@ -0,0 +1,60 @@ +# Naming + +The software is Swiss, and so are its names: both halves of the system are named after +places in the Alps that are, in one way or another, about moving a *lot* of something up a +steep mountain as efficiently as possible — usually by train. Throughput, in other words. + +| Part | Name | What it does | +| --- | --- | --- | +| Streaming / acquisition | **Jungfraujoch** | Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing. | +| Data processing | **Rugnux** | Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the [`rugnux`](RUGNUX.md) tool). | + +## Jungfraujoch + +The **Jungfraujoch** is a high mountain col in the Bernese Alps, the saddle (*Joch* is German +for "yoke" or "col") between the peaks **Jungfrau** and **Mönch**, at 3,466 m. It is the site of +the [High Altitude Research Station Jungfraujoch](https://www.hfsjg.ch/), whose long-running +atmospheric measurements are **co-operated by the Paul Scherrer Institute** — the same institute +that develops this software and the JUNGFRAU detector. + +The name is also a small piece of word-play. PSI's **JUNGFRAU** detector and DECTRIS's **EIGER** +detector are both named after Bernese Alps peaks (the famous trio is *Eiger*, *Mönch*, *Jungfrau*). +The Jungfraujoch — the pass *between* Jungfrau and Mönch — is where those two detector worlds meet. + +And it fits the theme of the whole project: the Jungfraujoch is reached by the **Jungfraubahn**, +whose terminus is the **highest railway station in Europe** (3,454 m, the "Top of Europe"). It is +the closest you can get to that summit in a genuinely *high-throughput* way — by train, moving +crowds up the mountain — which is exactly what the streaming side of this software does with +detector frames. + +**Pronunciation (German):** *Jungfraujoch* ≈ **YUNG-frow-yokh**. +"Jung" as in *young*, "frau" rhymes with *cow*, and the final "joch" ends in the guttural *ch* of +Scottish *loch* or German *Bach* — not a hard *k*. + +## Rugnux + +**Piz Rugnux** is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern +Switzerland. (*Piz* is the Romansh word for "peak".) It rises above the **Albula line** of the +**Rhaetian Railway** (*Rhätische Bahn*), part of the "Rhaetian Railway in the Albula / Bernina +Landscapes" — a **UNESCO World Heritage Site** (*Welterbe*). + +That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude +in very little horizontal distance, it corkscrews through a series of **helical (spiral) tunnels** +looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a +steep mountain efficiently — the same idea the data-processing side of this software is built +around: pushing a large volume of diffraction data through the analysis pipeline. + +So the theme is consistent — **Swiss mountains, trains, and throughput** — while keeping the two +subsystems clearly distinct: *Jungfraujoch* streams, *Rugnux* processes. + +**Pronunciation (Romansh):** *Piz Rugnux* ≈ **peets roo-NYOOKS**. +The "gn" is a soft palatal *ñ*, as in *canyon* or Italian *gnocchi*, not two separate sounds. + +## What is Romansh? + +**Romansh** (*Rumantsch*) is the **fourth national language of Switzerland**, alongside German, +French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind +in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the +canton of Graubünden. It survives in several regional idioms, brought together in a standard form +called *Rumantsch Grischun*. Naming the processing engine with a Romansh mountain is a small nod to +the least-spoken but no-less-Swiss corner of the country. diff --git a/_sources/OPENAPI.md.txt b/_sources/OPENAPI.md.txt new file mode 100644 index 00000000..a13a3280 --- /dev/null +++ b/_sources/OPENAPI.md.txt @@ -0,0 +1,13 @@ +# OpenAPI +## OpenAPI specs + +See document with detailed [OpenAPI specs](OPENAPI_SPECS.rst). + +## Python client +Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. +For convenience, we provide Python client as [jfjoch-client](https://pypi.org/project/jfjoch-client/) PyPi package. +To install the client you can use `pip` tool: +``` +pip install jfjoch-client +``` +See [API reference from the OpenAPI generator](python_client/README.md). \ No newline at end of file diff --git a/_sources/OPENAPI_SPECS.rst.txt b/_sources/OPENAPI_SPECS.rst.txt new file mode 100644 index 00000000..15bf4e7f --- /dev/null +++ b/_sources/OPENAPI_SPECS.rst.txt @@ -0,0 +1,4 @@ +OpenAPI specification +===================== + +See document with detailed `OpenAPI specs <_static/redoc-static.html>`_ generated with Redocly. \ No newline at end of file diff --git a/_sources/PIXEL_MASK.md.txt b/_sources/PIXEL_MASK.md.txt new file mode 100644 index 00000000..143f0fd5 --- /dev/null +++ b/_sources/PIXEL_MASK.md.txt @@ -0,0 +1,46 @@ +# Pixel mask + +## Mask format + +Jungfraujoch follows generally [NXmx format](https://manual.nexusformat.org/classes/applications/NXmx.html) format for pixel mask. +Pixel mask is described as 32-bit unsigned integer array of size the same as the image. +Conditions to mask pixel are described by setting a particular bit to one. This way it is possible to encode reason why pixel is included in the pixel mask, also for one pixel there can be multiple reasons encoded at the same time. + +Bit values are set as follows: + +Bit 0 - gap (pixel with no sensor) + +Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn't set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask) + +Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization) + +Bit 8 - user defined mask + +Bit 30 - module edge (only for PSI systems) + +Bit 31 - chip edge interpolated pixel (multipixel) + +## Custom user mask + +Jungfraujoch allows to upload custom user mask. This happens in two steps. First create mask in TIFF format: + +```python +import numpy as np +import tifffile as tiff + +# Create a 2068x2164 numpy array filled with zeros, with 32-bit unsigned integers +array = np.zeros((2068, 2164), dtype=np.uint32) + +# Mark the pixel (300, 400) with the value 1 +array[300, 400] = 1 + +# Save the array as a TIFF file +tiff.imwrite('mask.tiff', array) +``` + +Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8). + +Then upload the mask to Jungfraujoch server: +```shell +curl -v http:///config/user_mask.tiff -XPUT --data-binary @mask.tiff +``` diff --git a/_sources/REPOSITORIES.md.txt b/_sources/REPOSITORIES.md.txt new file mode 100644 index 00000000..b03b211f --- /dev/null +++ b/_sources/REPOSITORIES.md.txt @@ -0,0 +1,47 @@ +# Linux package repositories +For convenience, we are providing package repositories. With versions including and excluding CUDA linking. +We recommend to install Jungfraujoch viewer from `nocuda` repository and remaining packages from `cuda12`/`cuda13` repository. + +## RHEL based systems + +For RHEL systems we provide the following repositories: + +| RHEL version | CUDA | Repository file | +|--------------|------|-------------------------------------------------------------------------| +| 8.x | 12.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo | +| 8.x | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo | +| 9.x | 13.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda13.repo | +| 9.x | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-nocuda.repo | + +To install the repository, run: + +```bash +dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo +``` +Currently signing of RPMs is not supported, so the repository file needs to be manually modified to set `gpgcheck=0` +or installation must run with `--nogpgcheck`. + +We provide the following packages in the repository: +* jfjoch +* jfjoch-driver +* jfjoch-writer +* jfjoch-viewer + +## Ubuntu based systems + +For Ubuntu systems, we also provide the following repositories: +``` +sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc +echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list +sudo apt update +``` + +`$distribution` uses Ubuntu names `jammy` (22.04) and `noble` (24.04). `$component` can be set to `cuda13` and `nocuda`. + +We provide the following packages in the repository: +* jfjoch-jfjoch +* jfjoch-driver +* jfjoch-writer +* jfjoch-viewer + +Ubuntu packages are currently only going through a very limited testing. \ No newline at end of file diff --git a/_sources/RUGNUX.md.txt b/_sources/RUGNUX.md.txt new file mode 100644 index 00000000..a74c8dc9 --- /dev/null +++ b/_sources/RUGNUX.md.txt @@ -0,0 +1,238 @@ +# rugnux + +`rugnux` is the **offline** crystallographic data-analysis tool of Jungfraujoch — the +data-processing half of the system (see [Naming](NAMING.md) for where the name comes from). +It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, +geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the +results to a `_process.h5` file, plus reflection files (`.mtz`/`.cif`/`.hkl`) when merging is +requested. + +It runs the *same* analysis code as the online and interactive tools, just driven from the +command line over a file rather than a live detector stream. + +> **Note.** `rugnux` is under very active development. This page describes the tool and +> its options at a high level; the authoritative, always-current list of options is the program's +> own usage message — run `rugnux` with no arguments. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| [`jfjoch_viewer`](JFJOCH_VIEWER.md) | Interactive, on-screen exploration | Qt desktop application | Displayed on screen (results not saved to disk) | +| **`rugnux`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | + +Use `rugnux` to re-analyse data after acquisition, to experiment with processing +parameters, or to produce merged intensities for downstream structure solution. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. The CUDA build +provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`fft`); without CUDA +only the CPU `fftw` indexer is available. Spot finding, integration and scaling run on the CPU and +scale with the thread count (`-N`). + +## Input and output + +**Input** is a single Jungfraujoch HDF5 master file (NXmx-based). If the dataset already contains +stored spot lists, two-pass rotation indexing can reuse them instead of re-running spot finding on +the first pass. + +**Output** (controlled by `-o, --output-prefix`, default `output`): + +- `_process.h5` — NXmx-compliant HDF5 with derived metadata (spots, indexing, + integration, azimuthal integration, per-image statistics). See + [HDF5 / NeXus data format](HDF5.md) for the layout. Written by default only when **not** merging + (i.e. under `--no-merge`); add `--write-process-h5` to also write it when merging. +- Merging is **on by default** (`--no-merge` disables it). The merged reflections are written as + `.cif` (mmCIF — the default), or `.mtz` / `.hkl` depending on + `--scaling-output`. Both the mmCIF and the MTZ carry the **refined unit cell** (from rotation + indexing) and the **space group determined from systematic absences** (constrained to the indexed + lattice symmetry). No-reference scaling additionally emits per-iteration `_iterN_scale.dat`. + +Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the +console. By default the written resolution is trimmed automatically where CC1/2 falls off +(`--resolution-cutoff cc-logistic`, CC1/2 target 0.30); set `--scaling-high-resolution` to fix the +limit by hand, or `--resolution-cutoff off` to keep the full range. + +## Re-scaling and re-merging (`rugnux --scale`) + +The `--scale` mode re-scales and merges the *already-integrated* reflections stored in a +`_process.h5` file, without re-running spot finding or integration. Use it to re-merge quickly with a +different space group, resolution limit, anomalous setting or reference MTZ. It reuses the same +`-o/-N/-s/-e/-S/-A/-B/-z/--scaling-*` options as the full run, and (unlike the full pipeline) does +not run a space-group search, so pass `-S` for the correct symmetry. + +## Quick start + +### Rotation data + +Index, integrate, scale and merge a rotation sweep, fully de novo: + +``` +rugnux rotation_master.h5 \ + -o lyso_rot -N 32 \ + --scaling-high-resolution 1.4 +``` + +Because the dataset carries a rotation goniometer axis, it is processed as **rotation data by +default**: two-pass rotation indexing (index the sweep once, then process every frame against that +lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). Scaling +and merging run **by default** (for both rotation and stills; `--no-merge` turns them off); the unit +cell is taken from the rotation indexer and the space group is determined from systematic absences, +and both are written +into the merged `.cif`. + +Run **fully de novo** (no `-C`/`-S`) for the best result — supplying a cell or space group up front +tends to *degrade* low-symmetry cases. `--scaling-high-resolution` (set it to your expected +resolution) sharpens both the space-group search and the error model. To tune the first pass use +`--two-pass-rotation=100` (or `-R100` — the first-pass image count); to force the sweep to be +treated as independent stills use `--force-still`. + +After the per-frame scale-fulls step, rotation scaling applies two **correction surfaces**, **on by +default** (`--no-scaling-corrections` disables both): + +- **Decay** — a global Debye–Waller relative-*B* over the run, for the radiation damage that weakens + later frames more at high resolution (a resolution×time systematic the resolution-flat per-frame + scale cannot remove). It only engages when the total relative-*B* exceeds a physical floor (2 Ų). +- **Absorption** — a smooth multiplicative factor over the diffracted-beam direction in the goniometer + frame (path length through the crystal). Negligible at hard X-rays / thin crystals; it matters at + low photon energy. Its benefit shows up most on model-based metrics: a smooth absorption error + largely cancels among symmetry mates (little effect on the error model / ISa) but still biases the + intensities, so it measurably lowers *R*free. + +Both are **cross-validated** — fitted on even-numbered frames and kept only if they improve the +held-out odd-frame symmetry-equivalent agreement by a clear margin (and vice versa) — so where the +systematic is absent they are a no-op rather than a source of added noise; that is why they are safe +to leave on. They run on the GPU when one is present, at negligible cost. + +### Still / serial data + +A dataset with **no goniometer axis** (e.g. a serial grid scan) is processed as **independent +stills automatically** — no flag needed. Known-cell indexing with the GPU fast-feedback indexer, +then merge against a reference structure: + +``` +rugnux serial_master.h5 \ + -o lyso_serial -N 32 \ + -X ffbidx -C 79,79,38,90,90,90 -S 96 \ + --spot-sigma 4 \ + -z reference.mtz \ + --scaling-high-resolution 1.8 +``` + +`ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills. For +weak serial data, tightening spot finding with `--spot-sigma 4` typically raises the indexing rate +substantially. If a dataset *does* carry a goniometer axis but you want per-frame stills processing +anyway, add `--force-still`. + +## Command-line options + +General: + +| Option | Description | +| --- | --- | +| `-o, --output-prefix ` | Output file prefix (default: `output`) | +| `-N, --threads ` | Number of worker threads (default: all hardware threads) | +| `-s, --start-image ` | First image to process (default: 0) | +| `-e, --end-image ` | Last image to process (default: all) | +| `-t, --stride ` | Process every *n*-th image (default: 1) | +| `-v, --verbose` | Verbose output | + +Modes (default: full analysis — spot finding, indexing, integration and merging): + +| Option | Description | +| --- | --- | +| `--azint-only` | Only run azimuthal integration (no spot finding/indexing); writes `_process.h5` | +| `--scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) | + +Spot finding: + +| Option | Description | +| --- | --- | +| `--spot-sigma ` | Noise sigma level for spot finding (default: 3.0) | +| `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | +| `--spot-high-resolution ` | High-resolution limit for spot finding, Å (default: 1.5) | +| `--max-spots ` | Maximum spot count (default: 250) | +| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling/merging; overrides the dataset/master-file setting (default: use the dataset value) | + +Azimuthal integration (the radial profile behind the per-image ice-ring score): + +| Option | Description | +| --- | --- | +| `-q, --azim-q-spacing ` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) | +| `--azim-min-q ` | Minimum Q, 1/Å | +| `--azim-max-q ` | Maximum Q, 1/Å | +| `--azim-phi-bins ` | Number of azimuthal (phi) bins (default: 1) | +| `--polarization-correction ` | Enable/disable the azimuthal polarization correction | +| `--solid-angle-correction ` | Enable/disable the azimuthal solid-angle correction | + +Indexing: + +A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation +indexing) by default; a dataset without one is processed as independent stills. `--force-still` +overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request +rotation explicitly and pick the pass or lattice. + +| Option | Description | +| --- | --- | +| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | +| `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`) | +| `-S, --space-group ` | Space group number (used for indexing and scaling) | +| `-r, --refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) | +| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) | +| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) | +| `--redo-rotation-spots` | Redo spot finding for the two-pass rotation first pass | +| `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass | + +Indexer choice in brief: `ffbidx` (GPU) refines toward a **known cell** and is best for sparse +serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the +[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md) for the algorithms. + +Scaling and merging: + +| Option | Description | +| --- | --- | +| `--no-merge` | Skip scaling and merging (on by default); write only the per-image `_process.h5` | +| `-A, --anomalous` | Anomalous mode (keep Friedel pairs separate) | +| `-B, --refine-bfactor` | Refine a per-image B-factor (stills only) | +| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | +| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) | +| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption correction surfaces fitted on the fulls after scale-fulls (see below) | +| `--capture-uncertainty ` | rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise) | +| `--min-captured-fraction ` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) | +| `--scaling-high-resolution ` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) | +| `--resolution-cutoff ` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) | +| `--resolution-cc-target ` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) | +| `--resolution-shells ` | Number of resolution shells in the reported statistics table (default: 10) | +| `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02) | +| `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for `rot3d`, off otherwise) | +| `--reject-delta-cchalf ` | Drop images with ΔCC1/2 below mean − N·stddev (default: off) | +| `--min-image-cc ` | Per-image CC limit, percent (default: no limit) | +| `--mosaicity ` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed | +| `--scaling-iterations ` | Scaling iterations with no reference data (default: 3) | +| `--scaling-output ` | Reflection output format: `cif` (mmCIF, default) \| `mtz` \| `txt` | +| `-z, --reference-mtz ` | Reference MTZ (enables reference-driven scaling) | +| `--reference-column